diff --git a/contrib/ci-workers/README.md b/contrib/ci-workers/README.md deleted file mode 100644 index 8ce9dc764..000000000 --- a/contrib/ci-workers/README.md +++ /dev/null @@ -1,62 +0,0 @@ -# Zcash CI workers - -This folder contains the Ansible playbooks for configuring a fresh OS -installation for use as a Buildbot worker in Zcash's CI. - -# Criteria for Adding Workers - -a. Don't add workers until users complain about a problem on a platform that - doesn't yet have workers. However, if we anticipate many users will use a - platform, we may pre-emptively add an unsupported worker for it. - -b. Prioritize the platforms that seem to have the most users. - -c. When adding workers, start by adding workers for the "most common" variant of - any distro. Then if users later encounter problems with a sub-variant, we can - consider adding new workers at that point. Example: add Ubuntu Desktop before - Xubuntu, on the assumption the former has a larger population base, and the - latter only materially differs in the GUI. - -# Setting up a latent worker on Amazon EC2 - -1. Add a regular (non-latent) worker to the master.cfg for dev-ci.z.cash, and - deploy the changes. - - This enables the Ansible playbook to run to completion, ending in the worker - connecting to the master. - -2. Start a basic EC2 instance using the template AMI for the target OS. - - Choose the smallest instance size, it won't be used for building Zcash. - -3. Figure out which user to log into the instance with. - - E.g. for the Ubuntu template, use "ubuntu" instead of "root" - - If you get an Ansible error later with a message like "Failed to connect to - the host via ssh: Received message too long 1349281121\r\n", that means the - instance is sending a text string in response to the SSH connection, and the - Ansible protocol is balking. Try manually logging in with the same - credentials to diagnose. - -4. Create `inventory/hosts` containing the following: - - [zcash-ci-worker-unix] - some-name ansible_host= ansible_ssh_user= - -5. Run `ansible-playbook -e buildbot_worker_host_template=templates/host.ec2.j2 -i inventory/hosts unix.yml`, - passing in the worker's Buildbot name and password. - - After a successful run, the worker should be connected to dev-ci.z.cash and - visible in its worker list. - -6. Create an AMI from the instance. This is the worker AMI to put into the - master.cfg for dev-ci.z.cash. - - 16 GB of storage should be sufficient. - -7. SSH into the instance, and edit the worker config to connect to ci.z.cash. - -8. Create an AMI from the instance. This is the worker AMI to put into the - master.cfg for ci.z.cash. - - 16 GB of storage should be sufficient. - -9. Delete the instance (it is no longer needed). - -10. Edit the master.cfg to turn the new worker into a latent (using the new AMI - IDs), add it to the appropriate worker groups, set up new builders etc. - - Deploy this via the normal PR review process. diff --git a/contrib/ci-workers/ansible.cfg b/contrib/ci-workers/ansible.cfg deleted file mode 100644 index c58fea3c0..000000000 --- a/contrib/ci-workers/ansible.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[ssh_connection] -pipelining = True diff --git a/contrib/ci-workers/files/bashrc b/contrib/ci-workers/files/bashrc deleted file mode 100644 index aaad18b92..000000000 --- a/contrib/ci-workers/files/bashrc +++ /dev/null @@ -1,2 +0,0 @@ -export PATH=$HOME/venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -export EDITOR=vim diff --git a/contrib/ci-workers/grind.yml b/contrib/ci-workers/grind.yml deleted file mode 100644 index ef7e5758e..000000000 --- a/contrib/ci-workers/grind.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# Configure a Buildbot worker -- include: unix.yml - -- name: Install grind-specific worker dependencies - hosts: zcash-ci-worker-unix - become: true - - vars_files: - - vars/default.yml - - tasks: - - name: Get dependencies for distribution - include_vars: "{{ item }}" - with_first_found: - - files: - - "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" - - "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml" - - "vars/{{ ansible_distribution }}.yml" - - "vars/{{ ansible_os_family }}.yml" - skip: true - - - name: Install required packages - package: - name: "{{ item }}" - state: present - with_items: "{{ grind_deps }}" diff --git a/contrib/ci-workers/tasks/install-brew.yml b/contrib/ci-workers/tasks/install-brew.yml deleted file mode 100644 index 5faedad0b..000000000 --- a/contrib/ci-workers/tasks/install-brew.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -- name: Check if brew is installed - stat: - path: /usr/local/bin/brew - register: brew_check - -- name: Fail if brew is unavailable - fail: - msg: 'brew is not installed! Please install Homebrew: https://docs.brew.sh/Installation.html' - when: not brew_check.stat.exists diff --git a/contrib/ci-workers/tasks/install-pip.yml b/contrib/ci-workers/tasks/install-pip.yml deleted file mode 100644 index 8beff50ef..000000000 --- a/contrib/ci-workers/tasks/install-pip.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: Fetch pip installer - get_url: - url: https://bootstrap.pypa.io/get-pip.py - dest: /tmp/get-pip.py - -- name: Install pip - command: "{{ ansible_python.executable }} /tmp/get-pip.py" diff --git a/contrib/ci-workers/templates/buildbot-worker.plist.j2 b/contrib/ci-workers/templates/buildbot-worker.plist.j2 deleted file mode 100644 index 225c73b8a..000000000 --- a/contrib/ci-workers/templates/buildbot-worker.plist.j2 +++ /dev/null @@ -1,23 +0,0 @@ - - - - - Label - net.buildbot.worker - ProgramArguments - - {{ buildbot_worker_dir }}/venv/bin/buildbot-worker - start - {{ buildbot_worker_name }} - - WorkingDirectory - {{ buildbot_worker_dir }} - UserName - {{ buildbot_worker_user }} - KeepAlive - - NetworkState - - - - diff --git a/contrib/ci-workers/templates/buildbot-worker.service.j2 b/contrib/ci-workers/templates/buildbot-worker.service.j2 deleted file mode 100644 index 625323be6..000000000 --- a/contrib/ci-workers/templates/buildbot-worker.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Buildbot worker -Wants=network.target -After=network.target - -[Service] -Type=forking -PIDFile={{ buildbot_worker_dir }}/{{ buildbot_worker_name }}/twistd.pid -WorkingDirectory={{ buildbot_worker_dir }} -ExecStart={{ buildbot_worker_dir }}/venv/bin/buildbot-worker start {{ buildbot_worker_name }} -ExecReload={{ buildbot_worker_dir }}/venv/bin/buildbot-worker restart {{ buildbot_worker_name }} -ExecStop={{ buildbot_worker_dir }}/venv/bin/buildbot-worker stop {{ buildbot_worker_name }} -Restart=always -User={{ buildbot_worker_user }} - -[Install] -WantedBy=multi-user.target diff --git a/contrib/ci-workers/templates/host.ec2.j2 b/contrib/ci-workers/templates/host.ec2.j2 deleted file mode 100644 index dee692e02..000000000 --- a/contrib/ci-workers/templates/host.ec2.j2 +++ /dev/null @@ -1 +0,0 @@ -OS: {{ ansible_distribution }} {{ ansible_distribution_version }} diff --git a/contrib/ci-workers/templates/host.j2 b/contrib/ci-workers/templates/host.j2 deleted file mode 100644 index 65c4cb709..000000000 --- a/contrib/ci-workers/templates/host.j2 +++ /dev/null @@ -1,3 +0,0 @@ -OS: {{ ansible_distribution }} {{ ansible_distribution_version }} -Memory: {{ ansible_memtotal_mb }} MB -CPU: {{ ansible_processor if ansible_processor is string else ansible_processor[1] }} ({{ ansible_processor_cores }} cores) diff --git a/contrib/ci-workers/unix.yml b/contrib/ci-workers/unix.yml deleted file mode 100644 index cbb693401..000000000 --- a/contrib/ci-workers/unix.yml +++ /dev/null @@ -1,209 +0,0 @@ ---- -- name: Configure a Buildbot worker for Zcash CI - hosts: zcash-ci-worker-unix - become: true - gather_facts: False - - vars_files: - - vars/default.yml - - vars/buildbot.yml - - vars_prompt: - - name: "buildbot_worker_admin" - prompt: "Admin details" - default: "Zcash " - - name: "buildbot_worker_name" - prompt: "Buildbot worker name (provided by ZECC)" - private: no - - name: "buildbot_worker_password" - prompt: "Buildbot worker password (provided by ZECC)" - - pre_tasks: - - name: Install Python 2.7 for Ansible and Buildbot - raw: test -e /usr/bin/python || test -e /usr/bin/python2 || test -e /usr/bin/python2.7 || test -e /usr/local/bin/python2.7 || (test -e /usr/bin/apt && apt -qqy update && apt install -qqy python) || (test -e /usr/bin/dnf && dnf install -qqy python2) || (test -e /usr/sbin/pkg && pkg install -qqy python2) - register: output - changed_when: - - output.stdout != "" - - output.stdout != "\r\n" - - - name: Check if Python is in the configured location - raw: test -e {{ ansible_python_interpreter }} - ignore_errors: true - register: python_check - when: ansible_python_interpreter is defined - - - name: Fail if configured Python is unavailable - fail: - msg: "Python is not accessible at {{ ansible_python_interpreter }} on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary." - when: ansible_python_interpreter is defined and python_check.rc == 1 - - - name: Check if Python is in the default location - raw: test -e /usr/bin/python - ignore_errors: true - register: python_check - when: ansible_python_interpreter is undefined - - - name: Fail if default Python is unavailable - fail: - msg: Python is not accessible at /usr/bin/python on this host! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary. - when: ansible_python_interpreter is undefined and python_check.rc == 1 - - - name: Gathering Facts - setup: - tags: deps - - - name: Fail if Python is the wrong version - fail: - msg: "The Python binary at {{ ansible_python.executable }} is version {{ ansible_python_version }}! Please set the inventory variable 'ansible_python_interpreter' to the location of the Python 2.7 binary." - when: ansible_python.version.major != 2 or ansible_python.version.minor != 7 - - tasks: - - name: Get dependencies for distribution - include_vars: "{{ item }}" - with_first_found: - - files: - - "vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" - - "vars/{{ ansible_distribution }}-{{ ansible_distribution_major_version | int }}.yml" - - "vars/{{ ansible_distribution }}.yml" - - "vars/{{ ansible_os_family }}.yml" - skip: true - tags: deps - - - name: Collate dependencies - set_fact: - package_deps: "{{ buildbot_deps + fetch_deps + conf_deps + build_deps + link_deps + dist_deps }}" - python_modules: "{{ buildbot_modules + rpc_test_modules }}" - tags: deps - - - name: Install Homebrew [MacOSX] - include: tasks/install-brew.yml - when: ansible_distribution == 'MacOSX' - tags: deps - - - name: Update rolling release [Arch Linux] - pacman: - update_cache: yes - upgrade: yes - when: ansible_distribution == 'Archlinux' - tags: deps - - - name: Install required packages - package: - name: "{{ item }}" - state: present - with_items: "{{ package_deps }}" - become_user: "{{ ansible_ssh_user if ansible_distribution == 'MacOSX' else 'root' }}" - tags: deps - - - name: Install pip [CentOS, MacOSX] - include: tasks/install-pip.yml - when: ansible_distribution in ['CentOS', 'MacOSX'] - - - name: Install required Python system modules - pip: - name: "{{ item }}" - state: latest - executable: "{{ '/usr/local/bin/pip' if ansible_distribution == 'MacOSX' else omit }}" - with_items: "{{ system_modules }}" - - - name: Set up the Buildbot worker user - user: - name: "{{ buildbot_worker_user }}" - comment: Buildbot worker - shell: /bin/bash - state: present - - - name: Get absolute path to Buildbot worker home directory - command: echo ~ - register: homedir - become_user: "{{ buildbot_worker_user }}" - - - name: Save absolute path to Buildbot worker home directory - set_fact: - buildbot_worker_dir: "{{ homedir.stdout }}" - - - name: Install required Python modules - pip: - name: "{{ item }}" - state: latest - virtualenv: "~{{ buildbot_worker_user }}/venv" - virtualenv_command: "{{ '/usr/local/bin/virtualenv' if ansible_distribution == 'MacOSX' else omit }}" - with_items: "{{ python_modules }}" - become_user: "{{ buildbot_worker_user }}" - notify: restart buildbot-worker - - - name: Create Buildbot worker - command: > - ~{{ buildbot_worker_user }}/venv/bin/buildbot-worker create-worker ~/{{ buildbot_worker_name }} - {{ buildbot_master_host }}:{{ buildbot_master_port }} - {{ buildbot_worker_name|quote }} {{ buildbot_worker_password|quote }} - args: - creates: "~/{{ buildbot_worker_name }}/buildbot.tac" - become_user: "{{ buildbot_worker_user }}" - - - name: Set admin details for Buildbot worker - copy: - content: "{{ buildbot_worker_admin }}" - dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/admin" - owner: "{{ buildbot_worker_user }}" - group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" - mode: "0644" - - - name: Set host details for Buildbot worker - template: - src: "{{ buildbot_worker_host_template }}" - dest: "~{{ buildbot_worker_user }}/{{ buildbot_worker_name }}/info/host" - owner: "{{ buildbot_worker_user }}" - group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" - mode: "0644" - - - name: Install custom bashrc for virtualenv - copy: - src: bashrc - dest: "~{{ buildbot_worker_user }}/.bashrc" - owner: "{{ buildbot_worker_user }}" - group: "{{ omit if ansible_distribution == 'MacOSX' else buildbot_worker_user }}" - mode: "0644" - - - name: Copy Buildbot worker systemd service unit - template: - src: templates/buildbot-worker.service.j2 - dest: "/etc/systemd/system/buildbot-worker.service" - owner: root - group: root - mode: "0644" - when: ansible_distribution != 'MacOSX' - notify: reload systemd - - - name: Copy Buildbot worker launchd service unit - template: - src: templates/buildbot-worker.plist.j2 - dest: "/Library/LaunchDaemons/net.buildbot.worker.plist" - owner: root - group: wheel - mode: "0644" - when: ansible_distribution == 'MacOSX' - - - name: Start Buildbot worker - service: - name: buildbot-worker - state: started - enabled: yes - when: ansible_distribution != 'MacOSX' - - - name: Load Buildbot worker service [MacOSX] - command: launchctl load /Library/LaunchDaemons/net.buildbot.worker.plist - when: ansible_distribution == 'MacOSX' - - - name: Start Buildbot worker [MacOSX] - command: launchctl start net.buildbot.worker - when: ansible_distribution == 'MacOSX' - - handlers: - - name: restart buildbot-worker - service: - name: buildbot-worker - state: restarted - - - name: reload systemd - command: /bin/systemctl daemon-reload diff --git a/contrib/ci-workers/vars/Archlinux.yml b/contrib/ci-workers/vars/Archlinux.yml deleted file mode 100644 index 50e5577c3..000000000 --- a/contrib/ci-workers/vars/Archlinux.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -buildbot_deps: - - python2-pip -build_deps: - - cmake - - multilib/gcc - - make diff --git a/contrib/ci-workers/vars/CentOS.yml b/contrib/ci-workers/vars/CentOS.yml deleted file mode 100644 index f577af5c0..000000000 --- a/contrib/ci-workers/vars/CentOS.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -buildbot_deps: [] # Empty to remove python-pip -build_deps: - - bzip2 - - cmake - - gcc - - gcc-c++ - - make - - patch -dist_deps: - - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in - - python-devel - - redhat-rpm-config diff --git a/contrib/ci-workers/vars/Debian.yml b/contrib/ci-workers/vars/Debian.yml deleted file mode 100644 index b6a46f0cd..000000000 --- a/contrib/ci-workers/vars/Debian.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -build_deps: - - build-essential # Depends on g++, libc6-dev, make - - cmake -dist_deps: - - python-dev diff --git a/contrib/ci-workers/vars/Fedora.yml b/contrib/ci-workers/vars/Fedora.yml deleted file mode 100644 index 2a7351c69..000000000 --- a/contrib/ci-workers/vars/Fedora.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -build_deps: - - cmake - - gcc - - gcc-c++ - - make - - patch -dist_deps: - - pkgconfig # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in - - python-devel - - redhat-rpm-config diff --git a/contrib/ci-workers/vars/FreeBSD.yml b/contrib/ci-workers/vars/FreeBSD.yml deleted file mode 100644 index 4b1f01997..000000000 --- a/contrib/ci-workers/vars/FreeBSD.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -buildbot_deps: - - py27-pip -build_deps: - - cmake - - gcc - - gmake -dist_deps: - - bash - - pkgconf # Required until b556beda264308e040f8d88aca4f2f386a0183d9 is pulled in diff --git a/contrib/ci-workers/vars/MacOSX.yml b/contrib/ci-workers/vars/MacOSX.yml deleted file mode 100644 index 80b1ae608..000000000 --- a/contrib/ci-workers/vars/MacOSX.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -buildbot_deps: - - coreutils # For gnproc etc. -# Most are already installed -build_deps: - - cmake diff --git a/contrib/ci-workers/vars/Ubuntu.yml b/contrib/ci-workers/vars/Ubuntu.yml deleted file mode 100644 index 0d0f584b4..000000000 --- a/contrib/ci-workers/vars/Ubuntu.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -build_deps: - - build-essential # Depends on g++, libc6-dev, make - - cmake diff --git a/contrib/ci-workers/vars/buildbot.yml b/contrib/ci-workers/vars/buildbot.yml deleted file mode 100644 index 38e3fd25a..000000000 --- a/contrib/ci-workers/vars/buildbot.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -buildbot_worker_user: zcbbworker -buildbot_master_host: dev-ci.z.cash -buildbot_master_port: 9899 -buildbot_worker_host_template: templates/host.j2 diff --git a/contrib/ci-workers/vars/default.yml b/contrib/ci-workers/vars/default.yml deleted file mode 100644 index a0b0da9e6..000000000 --- a/contrib/ci-workers/vars/default.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# These variables can be overridden in distribution files. - -# Dependencies required to install Buildbot -buildbot_deps: - - python-pip # So we can install Python modules - -# Dependencies required to download files -fetch_deps: - - curl # For depends/ - - git - - wget # For zcutil/fetch-params.sh - -# Dependencies required to configure Zcash -conf_deps: - - autoconf - - automake - - m4 - - pkg-config - -# Dependencies required to compile Zcash -build_deps: - - cmake - - g++ - - gcc - - make - -# Dependencies required to link Zcash -link_deps: - - libtool - -# Additional distribution-specific dependencies -dist_deps: [] - -# Additional grind-specific dependencies -grind_deps: - - lcov - - valgrind - -# Python modules required on the system -system_modules: - - virtualenv - -# Python modules required for a Zcash Buildbot worker -buildbot_modules: - - pip # Needs to be updated first so Buildbot installs - - buildbot-worker - - pyflakes - -# Python modules required to run the Zcash RPC test suite -rpc_test_modules: - - pyblake2 - - pyzmq