diff --git a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml index 58cdd3f..736c573 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml @@ -1,26 +1,95 @@ --- -# tasks file for executor - - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create executor group become: yes ansible.builtin.group: @@ -33,7 +102,7 @@ name: "{{ executor_os_user }}" group: "{{ executor_os_group }}" groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" - state: "present" + state: present - name: Write Executor SystemD unit file become: true @@ -47,7 +116,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ executor_os_user }}" group: "{{ executor_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml b/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml index a3d886c..1c1744f 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml @@ -2,26 +2,98 @@ # tasks file for scheduler - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create scheduler group become: yes @@ -35,7 +107,7 @@ name: "{{ scheduler_os_user }}" group: "{{ scheduler_os_group }}" groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" - state: "present" + state: present - name: Write Scheduler SystemD unit file become: true @@ -49,7 +121,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ scheduler_os_user }}" group: "{{ scheduler_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml index 231a1cd..60ec67d 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml @@ -1,6 +1,8 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io - firewalld + - python3-firewall container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 b/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 new file mode 100644 index 0000000..bc0aff9 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 @@ -0,0 +1,5 @@ +deb http://de.archive.ubuntu.com/ubuntu jammy main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-updates main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-backports main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-security main universe +deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu jammy stable diff --git a/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 b/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 new file mode 100644 index 0000000..9dfc2a5 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 @@ -0,0 +1,5 @@ +deb http://de.archive.ubuntu.com/ubuntu noble main universe +deb http://de.archive.ubuntu.com/ubuntu noble-updates main universe +deb http://de.archive.ubuntu.com/ubuntu noble-backports main universe +deb http://de.archive.ubuntu.com/ubuntu noble-security main universe +deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable diff --git a/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml b/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml new file mode 100644 index 0000000..4f12ab8 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml @@ -0,0 +1,19 @@ +- name: Ensure /etc/apt/keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + +- name: Download Docker official GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + failed_when: false + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Configure apt retries + copy: + mode: 0444 diff --git a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml index 7e7f119..23e4109 100644 --- a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml @@ -1,26 +1,96 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Ensure directories exist - become: true ansible.builtin.file: state: "directory" path: "{{ item }}" @@ -32,18 +102,16 @@ - "/root/bin" - name: Write config files - become: true ansible.builtin.template: src: "{{ item }}.j2" dest: "{{ graphite_config_location }}//{{ item }}" - mode: "0644" + mode: "0664" loop: - "carbonapi.yaml" notify: - Restart carbonapi - name: Write Carbonapi Systemd unit files - become: true ansible.builtin.template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -58,7 +126,6 @@ - Restart carbonapi - name: Write Carbonapi-healthcheck Systemd unit files - become: true ansible.builtin.template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -73,15 +140,13 @@ - Restart carbonapi-healthcheck - name: Write carbonapi-healthcheck script - become: true ansible.builtin.template: src: "carbonapi-healthcheck.sh.j2" dest: "/root/bin/carbonapi-healthcheck.sh" mode: "0744" - name: Set proper selinux context for executable file - become: true - community.general.sefcontext: + ansible.builtin.sefcontext: target: "/root/bin/carbonapi-healthcheck.sh.j2" setype: bin_t state: present diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml index 9d32282..69b51f0 100644 --- a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml @@ -1,7 +1,12 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. +remove_packages: [] + packages: - - docker.io - - netcat + - netcat-openbsd - policycoreutils-python-utils - python3-selinux diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/epmon/tasks/main.yml b/cloudmon/ansible/project/roles/epmon/tasks/main.yml index 312480c..4139183 100644 --- a/cloudmon/ansible/project/roles/epmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/epmon/tasks/main.yml @@ -1,26 +1,96 @@ --- -# tasks file for epmon - - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create epmon group become: yes ansible.builtin.group: @@ -47,7 +117,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ epmon_os_user }}" group: "{{ epmon_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml b/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml index 17ec30f..076d958 100644 --- a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml @@ -7,24 +7,96 @@ paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Construct cluster nodes list ansible.builtin.set_fact: etcd_cluster: "{{ (etcd_cluster|default([])) + [(item + '=https://' + hostvars[item]['internal_address'] + ':2380')] }}" loop: "{{ play_hosts }}" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Ensure directories exist become: true diff --git a/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml b/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml index 3436c52..98a4451 100644 --- a/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml @@ -1,5 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - container_command: docker diff --git a/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml index b6e6d0f..a5caca4 100644 --- a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml @@ -1,64 +1,7 @@ -- name: Include OS-specific variables - include_vars: "{{ lookup('first_found', params) }}" - vars: - params: - files: "{{ distro_lookup_path }}" - paths: - - 'vars' - -- name: Install firewalld - ansible.builtin.package: - name: "{{ item }}" - state: "present" - loop: - - "{{ packages }}" - when: "ansible_facts.pkg_mgr != 'atomic_container'" - -- name: Enable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_services_enable }}" - notify: - - Reload firewalld - -- name: Disable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_services_disable }}" - notify: - - Reload firewalld - -- name: Enable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_ports_enable }}" - notify: - - Reload firewalld - -- name: Disable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_ports_disable }}" - notify: - - Reload firewalld - -- name: Disable iptables - ansible.builtin.service: - name: "iptables" - state: "stopped" - enabled: "false" - ignore_errors: "true" - -- name: Enable firewalld - ansible.builtin.service: - name: "firewalld" - state: "started" - enabled: "true" +--- +# Host firewall management is meaningless in a container (no service manager, +# no host firewall), so allow targets to opt out. Defaults to true, so +# existing VM-based deployments are unaffected. +- name: Manage host firewall + ansible.builtin.include_tasks: manage.yaml + when: cloudmon_manage_firewall | default(true) | bool diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml new file mode 100644 index 0000000..b6e6d0f --- /dev/null +++ b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml @@ -0,0 +1,64 @@ +- name: Include OS-specific variables + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: "{{ distro_lookup_path }}" + paths: + - 'vars' + +- name: Install firewalld + ansible.builtin.package: + name: "{{ item }}" + state: "present" + loop: + - "{{ packages }}" + when: "ansible_facts.pkg_mgr != 'atomic_container'" + +- name: Enable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_services_enable }}" + notify: + - Reload firewalld + +- name: Disable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_services_disable }}" + notify: + - Reload firewalld + +- name: Enable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_ports_enable }}" + notify: + - Reload firewalld + +- name: Disable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_ports_disable }}" + notify: + - Reload firewalld + +- name: Disable iptables + ansible.builtin.service: + name: "iptables" + state: "stopped" + enabled: "false" + ignore_errors: "true" + +- name: Enable firewalld + ansible.builtin.service: + name: "firewalld" + state: "started" + enabled: "true" diff --git a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml index 1b4d624..0a95f0f 100644 --- a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml @@ -2,41 +2,101 @@ # tasks file for globalmon - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" -# - name: Install required packages -# become: true -# ansible.builtin.package: -# state: present -# name: "{{ item }}" -# loop: -# - "{{ packages }}" -# when: "ansible_facts.pkg_mgr != 'atomic_container'" -# register: task_result -# until: task_result is success -# retries: 5 +- name: Include base repos role for Docker CE setup + ansible.builtin.include_role: + name: base.repos -# - name: Create globalmon group -# become: true -# ansible.builtin.group: -# name: "{{ globalmon_os_group }}" -# state: "present" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + +- name: Install required packages + ansible.builtin.package: + state: present + name: "{{ packages }}" + when: "ansible_facts.pkg_mgr != 'atomic_container'" + register: install_result + until: install_result is success + retries: 5 + +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' -# - name: Create globalmon user -# become: true -# ansible.builtin.user: -# name: "{{ globalmon_os_user }}" -# group: "{{ globalmon_os_group }}" -# groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" -# state: "present" +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' - name: Write globalmon SystemD unit file - become: true ansible.builtin.template: src: "globalmon.service.j2" dest: "{{ globalmon_systemd_unit_path }}" @@ -44,35 +104,26 @@ - Restart globalmon - name: Create required directories - become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" - # owner: "{{ globalmon_os_user }}" - # group: "{{ globalmon_os_group }}" + state: directory mode: "{{ item.mode | default(omit) }}" loop: - { dest: "{{ globalmon_config_dir }}", mode: "0755" } - { dest: "{{ globalmon_log_dir }}", mode: "0777" } - name: Write env - become: true ansible.builtin.template: src: "env_globalmon" dest: "{{ globalmon_env_path }}" - # owner: "{{ globalmon_os_user }}" - # group: "{{ globalmon_os_group }}" mode: "0644" notify: - Restart globalmon - name: Write globalmon config - become: true ansible.builtin.copy: content: "{{ globalmon_config | to_nice_yaml(indent=2, width=79) }}" dest: "{{ globalmon_config_path }}" - # group: "{{ globalmon_os_group }}" - # owner: "{{ globalmon_os_user }}" mode: "0644" when: - "globalmon_config is defined" @@ -80,12 +131,9 @@ - Restart globalmon - name: Write globalmon secure config - become: true ansible.builtin.copy: content: "{{ globalmon_secure_config | to_nice_yaml(indent=2, width=79) }}" dest: "{{ globalmon_secure_config_path }}" - # group: "{{ globalmon_os_group }}" - # owner: "{{ globalmon_os_user }}" mode: "0644" when: - "globalmon_secure_config is defined" diff --git a/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml b/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/grafana/tasks/main.yml b/cloudmon/ansible/project/roles/grafana/tasks/main.yml index 4ca57ab..24cf8e8 100644 --- a/cloudmon/ansible/project/roles/grafana/tasks/main.yml +++ b/cloudmon/ansible/project/roles/grafana/tasks/main.yml @@ -1,34 +1,104 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create Grafana group - become: true ansible.builtin.group: name: "{{ grafana_os_user }}" state: present - name: Create Grafana user for docker/podman - become: true ansible.builtin.user: name: "{{ grafana_os_user }}" group: "{{ grafana_os_group }}" @@ -36,8 +106,7 @@ state: present - name: Create required directories - become: true - file: + ansible.builtin.file: path: "{{ item.dest }}" state: directory owner: "{{ grafana_os_user }}" @@ -48,8 +117,7 @@ - {dest: "/etc/grafana/provisioning"} - name: Write Grafana config env file - become: true - template: + ansible.builtin.template: src: "env.j2" dest: "/etc/grafana/env" owner: "{{ grafana_os_user }}" @@ -59,8 +127,7 @@ - restart grafana - name: Write Grafana Image Renderer Systemd unit file - become: true - template: + ansible.builtin.template: src: "grafana-renderer.service.j2" dest: "/etc/systemd/system/grafana-renderer.service" when: @@ -70,8 +137,7 @@ - restart grafana-renderer - name: Write Grafana Systemd unit file - become: true - template: + ansible.builtin.template: src: "grafana.service.j2" dest: "/etc/systemd/system/grafana.service" notify: @@ -81,20 +147,18 @@ meta: flush_handlers - name: Make sure the Grafana service started - become: true - systemd: + ansible.builtin.systemd: state: started name: "grafana.service" - name: Wait for grafana container to listen - become: true - wait_for: + ansible.builtin.wait_for: host: 0.0.0.0 port: 3000 timeout: 120 - name: Wait for api/health to come up - uri: + ansible.builtin.uri: url: "http://0.0.0.0:3000/api/health" status_code: 200 register: result diff --git a/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml b/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml index 3436c52..98a4451 100644 --- a/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml @@ -1,5 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - container_command: docker diff --git a/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml index ac4e03c..11a44ea 100644 --- a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml @@ -1,25 +1,97 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create graphite sync group become: yes @@ -42,7 +114,7 @@ ansible.posix.authorized_key: user: "{{ graphite_sync_user }}" key: "{{ graphite_public_key }}" - state: "present" + state: present when: - "graphite_public_key is defined" - "graphite_sync_user is defined" @@ -100,7 +172,7 @@ minute: "0" hour: "2" job: > - bash -c 'echo "--- $(date) ---"; + bash -c 'echo "--- $(date) --"; find /opt/graphite/storage/whisper -type f -mtime +270 -name "*.wsp" -delete -print; find /opt/graphite/storage/whisper -depth -type d -empty -delete -print' >> /var/log/graphite-cleanup.log 2>&1 diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index da414a6..cc7e97b 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -1,7 +1,13 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. +remove_packages: [] + packages: - - docker.io - - netcat + - netcat-openbsd + # ansible.builtin.cron needs the crontab binary. - cron - - apparmor-profiles + container_command: docker diff --git a/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml index ec5c718..ac12375 100644 --- a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml @@ -1,29 +1,101 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Write Postgres Systemd unit file become: true - template: + ansible.builtin.template: src: "postgres.service.j2" dest: "/etc/systemd/system/postgres.service" mode: "0644" @@ -35,13 +107,13 @@ - name: Make sure the Postgres service started become: true - systemd: + ansible.builtin.systemd: state: started name: "postgres.service" - name: Wait for postgres container to listen become: true - wait_for: + ansible.builtin.wait_for: host: 0.0.0.0 port: 5432 timeout: 120 diff --git a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml index 4893bc6..48863bd 100644 --- a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml @@ -1,7 +1,11 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. +remove_packages: [] + packages: - - docker.io - python3-psycopg2 container_command: docker -postgresql_data_dir: "/var/lib/psql" diff --git a/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml new file mode 100644 index 0000000..5d23161 --- /dev/null +++ b/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml @@ -0,0 +1,19 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + - python3-psycopg2 + +container_command: docker +postgresql_data_dir: "/var/lib/psql" diff --git a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml index 98aa8b1..3477d19 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml @@ -7,18 +7,89 @@ paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Construct cluster etcd nodes list ansible.builtin.set_fact: etcd_cluster_nodes: "{{ (etcd_cluster_nodes|default([])) + [item] }}" @@ -31,6 +102,7 @@ when: "item != inventory_hostname" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create postgres group become: yes diff --git a/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml b/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml index b7340c6..1b1b64b 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml @@ -1,4 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - python3-psycopg2 diff --git a/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml index 3198eed..4b3e894 100644 --- a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml @@ -1,22 +1,99 @@ - name: Include OS-specific variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - 'vars' +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install container runtime ansible.builtin.package: name: "{{ distro_packages }}" state: present + register: install_result + until: install_result is success + retries: 5 # If we are on CoreOS we can't use the module and podman already exists when: "ansible_facts.pkg_mgr != 'atomic_container'" +- name: Clean up stale sockets after package changes + block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create StatsD directory ansible.builtin.file: path: "{{ statsd_config_location }}" - state: "directory" + state: directory recurse: true - name: Write StatsD config diff --git a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml index 6542952..112be38 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml @@ -1,3 +1,13 @@ +--- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. +remove_packages: [] + distro_packages: - - docker.io + - netcat-openbsd + - cron + - apparmor-profiles + container_command: docker diff --git a/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml b/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml index 4972b68..dffe868 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml @@ -1,3 +1,7 @@ +--- +remove_packages: [] + distro_packages: - podman + container_command: podman diff --git a/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml new file mode 100644 index 0000000..f4d535d --- /dev/null +++ b/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +distro_packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/playbooks_k8s/pre.yaml b/playbooks_k8s/pre.yaml new file mode 100644 index 0000000..526856d --- /dev/null +++ b/playbooks_k8s/pre.yaml @@ -0,0 +1,19 @@ +--- +# Pre-run playbook for the Kubernetes/pod-based cloudmon-deploy job. +# +# The `zuul-debian` eco2 pod runs as root but ships without `sudo`. +# Any task that uses `become: true` will fail with "sudo: not found" +# (exit 127) unless sudo is present. We install it via `raw` (no +# interpreter dependency) since we are already root. +# +# Then we bootstrap python, pip and virtualenv via the OTC global +# roles. Docker CE is installed in the main run play (see run.yaml). +- hosts: all + pre_tasks: + - name: Ensure sudo is available + raw: command -v sudo >/dev/null 2>&1 || (apt-get update && apt-get install -y sudo) + changed_when: false + roles: + - ensure-python + - ensure-pip + - ensure-virtualenv diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml new file mode 100644 index 0000000..d39c8c5 --- /dev/null +++ b/playbooks_k8s/run.yaml @@ -0,0 +1,184 @@ +--- +# Main run playbook for the Kubernetes/pod-based cloudmon-deploy job. +# +# The whole CloudMon stack is deployed on a single `zuul-debian` pod +# (no separate controller / worker split). Docker CE is installed +# inline here because there is no verified global ensure-docker role. +# +# Docker is needed because cloudmon's graphite/statsd/postgres +# provisioning playbooks deploy containerized components via +# ansible-runner. +# +# NOTE: the pod has no systemd as PID 1, so the docker/containerd +# daemons are started directly (background processes), not via the +# service or systemd modules. +- hosts: all + become: true + gather_facts: false + vars: + docker_ce_packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + + - name: Install prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + register: prereq_result + retries: 5 + until: prereq_result is success + + # Add Docker CE apt repository + - name: Ensure /etc/apt/keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + # Pod environment: known zuul-debian pod (Debian bookworm, amd64). + # No ansible_facts available — hardcode values. + - name: Download Docker official GPG key + ansible.builtin.get_url: + url: "https://download.docker.com/linux/debian/gpg" + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + + - name: Add Docker CE apt repository + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/docker-ce.list + mode: '0644' + content: >- + deb [arch=amd64 + signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/debian + bookworm stable + + # Must refresh apt cache after adding the Docker CE repo, otherwise + # `apt` has no idea about the `docker-ce` packages. + - name: Update apt cache after adding Docker CE repository + ansible.builtin.apt: + update_cache: true + + # Remove conflicting Docker packages that may be in the distro image + - name: Remove conflicting Docker packages + ansible.builtin.package: + state: absent + name: + - docker + - docker-engine + - docker.io + - containerd + - runc + failed_when: false + + # Install Docker CE + - name: Install Docker CE packages + ansible.builtin.apt: + name: "{{ docker_ce_packages }}" + state: present + register: docker_install + retries: 5 + until: docker_install is success + + # Clean up stale sockets after package changes + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + failed_when: false + + # The node runs systemd as PID 1 (zuul-debian-systemd label), so let it own + # the daemons. Starting them by hand would collide with the units the + # cloudmon roles later enable. Daemon settings (MTU, cgroupdriver, + # iptables/bridge) are seeded into /etc/docker/daemon.json by the pod spec. + - name: Start containerd via systemd + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + daemon_reload: true + + - name: Start dockerd via systemd + ansible.builtin.systemd: + name: docker + state: started + enabled: true + + - name: Wait for Docker socket to appear (up to 60s) + ansible.builtin.wait_for: + path: /var/run/docker.sock + timeout: 60 + register: docker_sock + ignore_errors: true + + # Only runs when the socket never appeared. wait_for above is + # non-fatal so these diagnostics can be collected before failing; + # previously it aborted the play, so they never ran on failure, and + # the fail below had no condition, so it always ran on success. + - name: Diagnose why the docker daemon did not start + when: docker_sock is failed + block: + - name: Check if dockerd process exists + ansible.builtin.command: + cmd: pgrep -a dockerd + register: dockerd_ps + changed_when: false + failed_when: false + + - name: Show dockerd log (last 100 lines) + ansible.builtin.command: + cmd: journalctl -u docker --no-pager -n 100 + register: dockerd_tail + changed_when: false + failed_when: false + + - name: Show containerd log (last 100 lines) + ansible.builtin.command: + cmd: journalctl -u containerd --no-pager -n 100 + register: containerd_tail + changed_when: false + failed_when: false + + - name: Fail with diagnostic context + ansible.builtin.fail: + msg: |- + Docker daemon failed to start after 60s. + dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} + docker journal: + {{ dockerd_tail.stdout | default('no journal output') }} + containerd journal: + {{ containerd_tail.stdout | default('no journal output') }} + +- hosts: all + become: true + pre_tasks: + # The node image ships none of these: ansible.builtin.pip imports + # `packaging` on the target, and cloudmon_venv_path makes it build a + # venv, which needs the venv/virtualenv tooling. + - name: Install Python packaging and venv tooling + ansible.builtin.apt: + name: + - python3-packaging + - python3-pip + - python3-venv + - virtualenv + state: present + register: py_prereq + retries: 5 + until: py_prereq is success + roles: + - install_cloudmon + - deploy_cloudmon_k8s diff --git a/roles/deploy_cloudmon_k8s/defaults/main.yaml b/roles/deploy_cloudmon_k8s/defaults/main.yaml new file mode 100644 index 0000000..129b59e --- /dev/null +++ b/roles/deploy_cloudmon_k8s/defaults/main.yaml @@ -0,0 +1,2 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +cloudmon_venv_path: "{{ ansible_user_dir }}/.venv" diff --git a/roles/deploy_cloudmon_k8s/tasks/main.yaml b/roles/deploy_cloudmon_k8s/tasks/main.yaml new file mode 100644 index 0000000..5e4b568 --- /dev/null +++ b/roles/deploy_cloudmon_k8s/tasks/main.yaml @@ -0,0 +1,50 @@ +--- +- name: Make inventory + ansible.builtin.template: + dest: "{{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory" + src: "inventory.j2" + +- name: Deploy + args: + chdir: "{{ ansible_user_dir }}/{{ zuul_work_dir }}" + executable: "/bin/bash" + ansible.builtin.shell: | + source {{ cloudmon_venv_path }}/bin/activate + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure {{ item }} provision + loop: + - graphite + - statsd + - postgres + register: cloudmon_provision + failed_when: false + +# Run one component per task and report before failing. Previously all three +# ran in a single shell block with no `set -e`, so rc came from the last +# command only and the output was never surfaced -- a failure showed up as a +# bare "non-zero return code" with nothing to debug. +- name: Show cloudmon provision output + ansible.builtin.debug: + msg: |- + {{ item.item }} rc={{ item.rc }} + stdout: + {{ item.stdout | default('') }} + stderr: + {{ item.stderr | default('') }} + loop: "{{ cloudmon_provision.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Fail if any cloudmon provision step failed + ansible.builtin.fail: + msg: >- + cloudmon provision failed for: + {{ cloudmon_provision.results | selectattr('rc', 'ne', 0) + | map(attribute='item') | list }} + when: cloudmon_provision.results | selectattr('rc', 'ne', 0) | list | length > 0 + +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure grafana provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure apimon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure epmon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure metrics processor provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure globalmon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure status dashboard provision diff --git a/roles/deploy_cloudmon_k8s/templates/inventory.j2 b/roles/deploy_cloudmon_k8s/templates/inventory.j2 new file mode 100644 index 0000000..99387a7 --- /dev/null +++ b/roles/deploy_cloudmon_k8s/templates/inventory.j2 @@ -0,0 +1,43 @@ +all: + hosts: + # The whole CloudMon stack is deployed on the single `zuul-debian` + # pod, so there is no separate worker node. Everything runs + # "localhost" from the pod's point of view. + localhost: + ansible_host: localhost + # Defining localhost explicitly loses Ansible's implicit local + # connection, and the pod runs no sshd, so this must be set. + ansible_connection: local + internal_address: localhost + vars: + # The pod has no service manager and no host firewall, so firewalld and + # iptables service management cannot work here. + cloudmon_manage_firewall: false + children: + # A group's `hosts` must be a mapping, not a list -- a list makes Ansible + # reject the whole inventory ("requires a dictionary"), after which every + # play matches no hosts. + statsd: + hosts: + localhost: + graphite: + hosts: + localhost: + grafana: + hosts: + localhost: + schedulers: + hosts: + localhost: + executors: + hosts: + localhost: + epmons: + hosts: + localhost: + globalmons: + hosts: + localhost: + postgres: + hosts: + localhost: diff --git a/zuul.yaml b/zuul.yaml index 2e1f966..a913806 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -20,18 +20,43 @@ - name: worker label: ubuntu-jammy +# Kubernetes/pod-based deployment. Instead of provisioning two ubuntu-jammy +# VMs (controller + worker) this runs the whole CloudMon stack on a single +# `zuul-debian` pod. The pod runs as root but ships without sudo and has no +# multi-node SSH setup, so it uses its own playbooks (playbooks_k8s) that +# bootstrap sudo + python and deploy on one node. +- job: + name: cloudmon-deploy-k8s + abstract: true + description: | + Deploy CloudMon on Kubernetes pods + pre-run: playbooks_k8s/pre.yaml + run: playbooks_k8s/run.yaml + files: + - ^cloudmon/* + vars: + python_version: "3.11" + +- job: + name: cloudmon-deploy-zuul-debian + parent: cloudmon-deploy-k8s + nodeset: + nodes: + - name: zuul-debian + label: zuul-debian-systemd + - project: merge-mode: squash-merge default-branch: main check: jobs: - otc-tox-pep8 - - otc-tox-py39 + - otc-tox-py311 - otc-tox-docs - - cloudmon-deploy-ubuntu-jammy + - cloudmon-deploy-zuul-debian gate: jobs: - otc-tox-pep8 - - otc-tox-py39 + - otc-tox-py311 - otc-tox-docs - - cloudmon-deploy-ubuntu-jammy + - cloudmon-deploy-zuul-debian