From 366b2a25ca91a1372554574c341281f0773874b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Busse?= Date: Tue, 9 Mar 2021 20:15:25 +0100 Subject: [PATCH] testsuite: Use more ansible for setup --- tests/ansible-hadoop/README.md | 10 +++ tests/{ => ansible-hadoop}/hbase-create.yml | 0 tests/{ => ansible-hadoop}/hdfs-create.yml | 0 tests/{ => ansible-hadoop}/inventory.yml | 0 .../roles}/hbase/defaults/main.yml | 5 +- .../roles}/hbase/files/hbase-config.sh | 6 +- .../ansible-hadoop/roles/hbase/tasks/main.yml | 31 +++++++++ .../roles/hdfs/defaults/main.yml | 15 +++++ .../roles}/hdfs/files/hadoop-config.sh | 35 +++++------ .../ansible-hadoop/roles/hdfs/tasks/main.yml | 37 +++++++++++ tests/{ => ansible-hadoop}/setup.sh | 0 tests/hbase/tasks/main.yml | 30 --------- tests/hdfs/defaults/main.yml | 4 -- tests/hdfs/tasks/main.yml | 36 ----------- tests/test_hbase_exporter.sh | 63 +++++++------------ 15 files changed, 138 insertions(+), 134 deletions(-) create mode 100644 tests/ansible-hadoop/README.md rename tests/{ => ansible-hadoop}/hbase-create.yml (100%) rename tests/{ => ansible-hadoop}/hdfs-create.yml (100%) rename tests/{ => ansible-hadoop}/inventory.yml (100%) rename tests/{ => ansible-hadoop/roles}/hbase/defaults/main.yml (63%) rename tests/{ => ansible-hadoop/roles}/hbase/files/hbase-config.sh (89%) create mode 100644 tests/ansible-hadoop/roles/hbase/tasks/main.yml create mode 100644 tests/ansible-hadoop/roles/hdfs/defaults/main.yml rename tests/{ => ansible-hadoop/roles}/hdfs/files/hadoop-config.sh (67%) create mode 100644 tests/ansible-hadoop/roles/hdfs/tasks/main.yml rename tests/{ => ansible-hadoop}/setup.sh (100%) delete mode 100644 tests/hbase/tasks/main.yml delete mode 100644 tests/hdfs/defaults/main.yml delete mode 100644 tests/hdfs/tasks/main.yml diff --git a/tests/ansible-hadoop/README.md b/tests/ansible-hadoop/README.md new file mode 100644 index 0000000..16e2be0 --- /dev/null +++ b/tests/ansible-hadoop/README.md @@ -0,0 +1,10 @@ +# ansible-hadoop + +## About +Bootstraps a HDFS/Hbase cluster with Ansible + +## Usage +``` +$ ansible-playbook -i inventory.yml -e hadoop_path="/tmp" -e hdfs_cluster_id="test-cluster" hdfs-create.yml +$ ansible-playbook -i inventory.yml -e "hbase_path=/tmp" hbase-create.yml +``` diff --git a/tests/hbase-create.yml b/tests/ansible-hadoop/hbase-create.yml similarity index 100% rename from tests/hbase-create.yml rename to tests/ansible-hadoop/hbase-create.yml diff --git a/tests/hdfs-create.yml b/tests/ansible-hadoop/hdfs-create.yml similarity index 100% rename from tests/hdfs-create.yml rename to tests/ansible-hadoop/hdfs-create.yml diff --git a/tests/inventory.yml b/tests/ansible-hadoop/inventory.yml similarity index 100% rename from tests/inventory.yml rename to tests/ansible-hadoop/inventory.yml diff --git a/tests/hbase/defaults/main.yml b/tests/ansible-hadoop/roles/hbase/defaults/main.yml similarity index 63% rename from tests/hbase/defaults/main.yml rename to tests/ansible-hadoop/roles/hbase/defaults/main.yml index 83946f7..27cad28 100644 --- a/tests/hbase/defaults/main.yml +++ b/tests/ansible-hadoop/roles/hbase/defaults/main.yml @@ -1,4 +1,7 @@ hbase_version: "0.96.1.1" hbase_archive_file: "hbase-{{ hbase_version }}-hadoop2-bin.tar.gz" hbase_archive_file_checksum256: "7334e7da0b655ab02cfc64454c3d2e93a4c584efbde2dfd37915b9530d1643f8" -hbase_archive_url: "https://archive.apache.org/dist/hbase/hbase-{{ hbase_version }}/{{ hbase_archive }}" +hbase_archive_url: "https://archive.apache.org/dist/hbase/hbase-{{ hbase_version }}/{{ hbase_archive_file }}" +hbase_config_path: "/tmp/hbase/conf" +hbase_rootdir: "/tmp/hbase-root" +hbase_datadir: "/tmp/hbase-data" diff --git a/tests/hbase/files/hbase-config.sh b/tests/ansible-hadoop/roles/hbase/files/hbase-config.sh similarity index 89% rename from tests/hbase/files/hbase-config.sh rename to tests/ansible-hadoop/roles/hbase/files/hbase-config.sh index bf9f481..5b0399a 100755 --- a/tests/hbase/files/hbase-config.sh +++ b/tests/ansible-hadoop/roles/hbase/files/hbase-config.sh @@ -8,11 +8,11 @@ HBASE_DIR="hbase-${HBASE_VERSION}-hadoop2" #HBASE_URL="https://downloads.apache.org/hbase/${HBASE_VERSION}/${HBASE_FILE}" HBASE_URL="https://archive.apache.org/dist/hbase/hbase-${HBASE_VERSION}/${HBASE_FILE}" HBASE_FILE_CKSUM="1625453f839f7d8c86078a131af9731f6df28c59e58870db84913dcbc640d430253134a825de7cec247ea1f0cf232435765e00844ee2e4faf31aeb356955c478" -HBASE_CONFIG_TEMPLATE="hbase/conf/hbase-site.xml.j2" -HBASE_TEST_SUITE_EXECUTABLE="hbase/bin/hbase" +HBASE_PATH="/tmp" +HBASE_CONFIG_TEMPLATE="${HBASE_PATH}/hbase/conf/hbase-site.xml.j2" SCRIPT_PATH=$(dirname "$0") -source $SCRIPT_PATH/../../setup.sh +source $SCRIPT_PATH/../../../setup.sh create_hbase_config_template() { read -r -d '' CONFIG < fs.defaultFS - hdfs://{{ cluster_id }} + hdfs://{{ hdfs_cluster_id }} dfs.journalnode.edits.dir @@ -57,38 +51,38 @@ create_hdfs_config_template() { dfs.nameservices - {{ cluster_id }} + {{ hdfs_cluster_id }} - dfs.ha.namenodes.{{ cluster_id }} + dfs.ha.namenodes.{{ hdfs_cluster_id }} nn1,nn2 fs.defaultFS - hdfs://{{ cluster_id }} + hdfs://{{ hdfs_cluster_id }} - dfs.namenode.rpc-address.{{ cluster_id }}.nn1 + dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn1 {{ namenode_1 }}:8020 - dfs.namenode.rpc-address.{{ cluster_id }}.nn2 + dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn2 {{ namenode_2 }}:8020 - dfs.namenode.http-address.{{ cluster_id }}.nn1 + dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1 {{ namenode_1 }}:50070 - dfs.namenode.http-address.{{ cluster_id }}.nn2 + dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2 {{ namenode_2}}:50070 - dfs.namenode.http-address.{{ cluster_id }}.nn1 + dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1 {{ namenode_1 }}:9870 - dfs.namenode.http-address.{{ cluster_id }}.nn2 + dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2 {{ namenode_2 }}:9870 @@ -118,4 +112,3 @@ HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config_template) write_file ${HDFS_CONFIG_TEMPLATE} "${HDFS_CONFIG}" write_file ${HDFS_CONFIG_TEMPLATE_CORE} "${HDFS_CONFIG_CORE}" write_file ${HDFS_CONFIG_TEMPLATE_MAPRED} "${HDFS_CONFIG_MAPRED}" -write_file ${HDFS_CONFIG_DATANODES} "localhost" diff --git a/tests/ansible-hadoop/roles/hdfs/tasks/main.yml b/tests/ansible-hadoop/roles/hdfs/tasks/main.yml new file mode 100644 index 0000000..743a43e --- /dev/null +++ b/tests/ansible-hadoop/roles/hdfs/tasks/main.yml @@ -0,0 +1,37 @@ +--- + + - name: Download Hadoop archive if not existent + get_url: + url: "{{ hadoop_archive_url }}" + dest: "{{ hadoop_path }}/{{ hadoop_archive_file }}" + mode: 0600 + checksum: sha256:{{ hadoop_archive_file_checksum256 }} + run_once: true + delegate_to: localhost + + - name: Extract archive + ansible.builtin.unarchive: + src: "{{ hadoop_path }}/{{ hadoop_archive_file }}" + dest: "{{ hadoop_path }}" + + - name: Recursively remove directory + ansible.builtin.file: + path: "{{ hadoop_path }}/{{ hadoop_dir }}" + state: absent + + - name: Rename dir + command: mv -f {{ hadoop_path }}/{{ hadoop_archive_dir }} {{ hadoop_path }}/{{ hadoop_dir }} + + - name: Write config templates + ansible.builtin.command: + cmd: "roles/hdfs/files/hadoop-config.sh" + delegate_to: localhost + + - name: Create core-site.xml + template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/core-site.xml mode=0700 + + - name: Create hdfs-site.xml + template: src={{ hadoop_config_path}}/hdfs-site.xml.j2 dest={{ hadoop_config_path}}/hdfs-site.xml mode=0700 + + - name: Create mapred-site.xml + template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/mapred-site.xml mode=0700 diff --git a/tests/setup.sh b/tests/ansible-hadoop/setup.sh similarity index 100% rename from tests/setup.sh rename to tests/ansible-hadoop/setup.sh diff --git a/tests/hbase/tasks/main.yml b/tests/hbase/tasks/main.yml deleted file mode 100644 index c666903..0000000 --- a/tests/hbase/tasks/main.yml +++ /dev/null @@ -1,30 +0,0 @@ ---- - - - name: Download HBase archive - get_url: - url: "{{ hbase_archive_url }}" - dest: "{{ script_path }}" - mode: 0600 - checksum: sha256:{{ hbase_archive_file_checksum256 }} - run_once: true - delegate_to: localhost - - - name: Copy archive - copy: - src: "{{ script_path }}/{{ hbase_archive }}" - dest: "/tmp" - remote_src: yes - - - name: Extract archive - ansible.builtin.unarchive: - src: "/tmp/{{ hbase_archive }}" - dest: "/tmp" - remote_src: yes - - - name: Write config template - ansible.builtin.command: - cmd: "{{ script_path }}/hbase/files/hbase-config.sh" - delegate_to: localhost - - - name: Create hbase-site.xml - template: src={{ script_path }}/{{ hbase_config_path }}/hbase-site.xml.j2 dest={{ script_path }}/{{ hbase_config_path }}/hbase-site.xml mode=0700 diff --git a/tests/hdfs/defaults/main.yml b/tests/hdfs/defaults/main.yml deleted file mode 100644 index d6e1405..0000000 --- a/tests/hdfs/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -hadoop_version: "2.10.0" -hadoop_archive_file: "hadoop-{{ hadoop_version }}.tar.gz" -hadoop_archive_file_checksum256: "273d5fa1d479d0bb96759b16cf4cbd6ba3e7f863a0778cbae55ab83417e961f0" -hadoop_archive_url: "https://artfiles.org/apache.org/hadoop/common/hadoop-{{ hadoop_version }}}/{{ hadoop_archive_file }}" diff --git a/tests/hdfs/tasks/main.yml b/tests/hdfs/tasks/main.yml deleted file mode 100644 index b475cc5..0000000 --- a/tests/hdfs/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - - - name: Download Hadoop archive - get_url: - url: "{{ hadoop_archive_url }}" - dest: "{{ script_path }}" - mode: 0600 - checksum: sha256:{{ hadoop_archive_file_checksum256 }} - run_once: true - delegate_to: localhost - - - name: Copy archive - copy: - src: "{{ script_path }}/{{ hadoop_archive }}" - dest: "/tmp" - remote_src: yes - - - name: Extract archive - ansible.builtin.unarchive: - src: "/tmp/{{ hadoop_archive }}" - dest: "/tmp" - remote_src: yes - - - name: Write config template - ansible.builtin.command: - cmd: "{{ script_path }}/hdfs/files/hadoop-config.sh" - delegate_to: localhost - - - name: Create core-site.xml - template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/core-site.xml mode=0700 - - - name: Create hdfs-site.xml - template: src={{ hdfs_config_path}}/hdfs-site.xml.j2 dest={{ hdfs_config_path}}/hdfs-site.xml mode=0700 - - - name: Create mapred-site.xml - template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/mapred-site.xml mode=0700 diff --git a/tests/test_hbase_exporter.sh b/tests/test_hbase_exporter.sh index e9dfc91..edd1728 100644 --- a/tests/test_hbase_exporter.sh +++ b/tests/test_hbase_exporter.sh @@ -1,18 +1,10 @@ #!/usr/bin/env bash +SETUP_HBASE=false +SETUP_HADOOP=true HBASE_TIME_STARTUP=15 HBASE_EXPORTER_TIME_STARTUP=60 -HBASE_CMD="./bin/hbase-daemon.sh --config conf start master" -HDFS_FORMAT=false HADOOP_CONFIG_DIR="hadoop/etc/hadoop" -HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode" -HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} datanode" -HDFS_CMD_ZKFC="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} start zkfc" -HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -format" -HDFS_CMD_FORMAT_HA="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -initializeSharedEdits" - -SCRIPT_PATH=$(dirname "$0") -source $SCRIPT_PATH/setup.sh setup_suite() { if [ "FreeBSD" = "$(uname)" ]; then @@ -23,44 +15,37 @@ setup_suite() { export HADOOP_PREFIX="$(pwd)/hadoop" - # Setup HBase - if ! ansible-playbook -i inventory.yml \ - -e "hdfs_config_path=${HADOOP_PREFIX}/etc/hadoop" \ - -e "script_path=$(pwd)" \ - hbase-create.yml; then + cd ansible-hadoop/ || exit 1 - printf "Failed to setup HBase to run test suite\n" - exit 1 + SCRIPT_PATH=$(dirname "$0") + source $SCRIPT_PATH/ansible-hadoop/setup.sh + + # Setup Hadoop + if [ true = "$SETUP_HADOOP" ]; then + if ! ansible-playbook -i inventory.yml \ + -e hadoop_path="/tmp" \ + -e hdfs_cluster_id="test-cluster" \ + hdfs-create.yml; then + + printf "Failed to setup HDFS to run test suite\n" + exit 1 + fi fi - # Setup HDFS - if ! ansible-playbook -i inventory.yml \ - -e "hdfs_config_path=$(pwd)/hadoop/etc/hadoop" \ - -e "script_path=$(pwd)" \ - hdfs-create.yml; then + # Setup HBase + if [ true = "$SETUP_HBASE" ]; then + if ! ansible-playbook -i inventory.yml \ + -e hbase_path="/tmp" \ + hbase-create.yml; then - printf "Failed to setup HDFS to run test suite\n" - exit 1 + printf "Failed to setup HBase to run test suite\n" + exit 1 + fi fi # Start hdfs - if [ true = "$HDFS_FORMAT" ]; then - printf "Formatting %s\n" "$1" - r=run $HDFS_CMD_FORMAT "HDFS FORMAT" - r=run $HDFS_CMD_FORMAT_HA "HDFS FORMAT HA" - fi - - run "$HDFS_CMD_NAMENODE" "HDFS Namenode" - run "$HDFS_CMD_DATANODE" "HDFS Datanode" # Start HBase - cd hbase/ || exit - run "$HBASE_CMD" "HBASE" - if [[ "$r" == *"Stop it first."* ]]; then - printf "HBase is already running. Stop it manually first, then run script again" - exit 1 - fi - sleep $HBASE_TIME_STARTUP # Start exporter run_exporter