testsuite: Use more ansible for setup

tests/ansible-hadoop/README.md

@@ -0,0 +1,10 @@
+# ansible-hadoop
+
+## About
+Bootstraps a HDFS/Hbase cluster with Ansible
+
+## Usage
+```
+$ ansible-playbook -i inventory.yml -e hadoop_path="/tmp" -e hdfs_cluster_id="test-cluster" hdfs-create.yml
+$ ansible-playbook -i inventory.yml -e "hbase_path=/tmp" hbase-create.yml
+```

tests/ansible-hadoop/roles/hbase/defaults/main.yml

@@ -1,4 +1,7 @@
 hbase_version: "0.96.1.1"
 hbase_archive_file: "hbase-{{ hbase_version }}-hadoop2-bin.tar.gz"
 hbase_archive_file_checksum256: "7334e7da0b655ab02cfc64454c3d2e93a4c584efbde2dfd37915b9530d1643f8"
-hbase_archive_url: "https://archive.apache.org/dist/hbase/hbase-{{ hbase_version }}/{{ hbase_archive }}"
+hbase_archive_url: "https://archive.apache.org/dist/hbase/hbase-{{ hbase_version }}/{{ hbase_archive_file }}"
+hbase_config_path: "/tmp/hbase/conf"
+hbase_rootdir: "/tmp/hbase-root"
+hbase_datadir: "/tmp/hbase-data"

tests/ansible-hadoop/roles/hbase/files/hbase-config.sh

@@ -8,11 +8,11 @@ HBASE_DIR="hbase-${HBASE_VERSION}-hadoop2"
 #HBASE_URL="https://downloads.apache.org/hbase/${HBASE_VERSION}/${HBASE_FILE}"
 HBASE_URL="https://archive.apache.org/dist/hbase/hbase-${HBASE_VERSION}/${HBASE_FILE}"
 HBASE_FILE_CKSUM="1625453f839f7d8c86078a131af9731f6df28c59e58870db84913dcbc640d430253134a825de7cec247ea1f0cf232435765e00844ee2e4faf31aeb356955c478"
-HBASE_CONFIG_TEMPLATE="hbase/conf/hbase-site.xml.j2"
+HBASE_PATH="/tmp"
-HBASE_TEST_SUITE_EXECUTABLE="hbase/bin/hbase"
+HBASE_CONFIG_TEMPLATE="${HBASE_PATH}/hbase/conf/hbase-site.xml.j2"
 
 SCRIPT_PATH=$(dirname "$0")
-source $SCRIPT_PATH/../../setup.sh
+source $SCRIPT_PATH/../../../setup.sh
 
 create_hbase_config_template() {
     read -r -d '' CONFIG <<EOF

tests/ansible-hadoop/roles/hbase/tasks/main.yml

@@ -0,0 +1,31 @@
+---
+
+  - name: Download HBase archive locally
+    get_url:
+      url: "{{ hbase_archive_url }}"
+      dest: "/tmp/hbase.tar.gz"
+      mode: 0600
+      checksum: sha256:{{ hbase_archive_file_checksum256 }}
+    run_once: true
+    delegate_to: localhost
+
+  - name: Copy and extract archive
+    ansible.builtin.unarchive:
+      src: "/tmp/hbase.tar.gz"
+      dest: "/tmp"
+
+  - name: Recursively remove directory
+    ansible.builtin.file:
+      path: /tmp/hbase
+      state: absent
+
+  - name: Rename dir
+    command: mv -f /tmp/hbase-0.96.1.1-hadoop2 /tmp/hbase
+
+  - name: Write config template
+    ansible.builtin.command:
+      cmd: "hbase/files/hbase-config.sh"
+    delegate_to: localhost
+
+  - name: Create hbase-site.xml
+    template: src={{ hbase_config_path }}/hbase-site.xml.j2 dest={{ hbase_config_path }}/hbase-site.xml mode=0700

tests/ansible-hadoop/roles/hdfs/defaults/main.yml

@@ -0,0 +1,15 @@
+hadoop_version: "2.10.0"
+hadoop_dir: "hadoop"
+hadoop_archive_file: "hadoop-{{ hadoop_version }}.tar.gz"
+hadoop_archive_dir: "hadoop-{{ hadoop_version }}"
+hadoop_archive_file_checksum256: "131750c258368be4baff5d4a83b4de2cd119bda3774ed26d1d233b6fdf33f07f"
+hadoop_archive_url: "https://archive.apache.org/dist/hadoop/common/hadoop-{{ hadoop_version }}/{{ hadoop_archive_file }}"
+hadoop_config_path: "{{ hadoop_path }}/hadoop/etc/hadoop"
+hdfs_cmd_namenode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode"
+hdfs_cmd_datanode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs datanode"
+hdfs_cmd_journalnode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs journalnode"
+hdfs_cmd_zkfc: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs start zkfc"
+hdfs_cmd_format: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode -format"
+hdfs_cmd_format_ha: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode -initializeSharedEdits"
+namenode_1: "namenode-1"
+namenode_2: "namenode_2"

tests/ansible-hadoop/roles/hdfs/files/hadoop-config.sh

@@ -2,18 +2,12 @@
 
 set -ueo pipefail
 
-HADOOP_VERSION="2.10.1"
+HDFS_CONFIG_TEMPLATE="/tmp/hadoop/etc/hadoop/hdfs-site.xml.j2"
-HADOOP_FILE="hadoop-$HADOOP_VERSION.tar.gz"
+HDFS_CONFIG_TEMPLATE_CORE="/tmp/hadoop/etc/hadoop/core-site.xml.j2"
-HADOOP_URL="https://artfiles.org/apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}"
+HDFS_CONFIG_TEMPLATE_MAPRED="/tmp/hadoop/etc/hadoop/mapred-site.xml.j2"
-HADOOP_FILE_CKSUM="2460e02cd1f80dfed7a8981bbc934c095c0a341435118bec781fd835ec2ebdc5543a03d92d24f2ddeebdfe1c2c460065ba1d394ed9a73cbb2020b40a8d8b5e07"
-HDFS_CONFIG_TEMPLATE="hadoop/etc/hadoop/hdfs-site.xml.j2"
-HDFS_CONFIG_TEMPLATE_CORE="hadoop/etc/hadoop/core-site.xml.j2"
-HDFS_CONFIG_TEMPLATE_MAPRED="hadoop/etc/hadoop/mapred-site.xml.j2"
-HDFS_CONFIG_DATANODES="localhost"
-HDFS_TEST_SUITE_EXECUTABLE="hadoop/bin/hdfs"
 
 SCRIPT_PATH=$(dirname "$0")
-source $SCRIPT_PATH/../../setup.sh
+source setup.sh
 
 create_hdfs_core_config_template() {
     #printf "Writing HDFS core-site.xml config\n"
@@ -23,7 +17,7 @@ create_hdfs_core_config_template() {
 <configuration>
   <property>
     <name>fs.defaultFS</name>
-    <value>hdfs://{{ cluster_id }}</value>
+    <value>hdfs://{{ hdfs_cluster_id }}</value>
   </property>
   <property>
     <name>dfs.journalnode.edits.dir</name>
@@ -57,38 +51,38 @@ create_hdfs_config_template() {
 <configuration>
   <property>
     <name>dfs.nameservices</name>
-    <value>{{ cluster_id }}</value>
+    <value>{{ hdfs_cluster_id }}</value>
   </property>
   <property>
-    <name>dfs.ha.namenodes.{{ cluster_id }}</name>
+    <name>dfs.ha.namenodes.{{ hdfs_cluster_id }}</name>
     <value>nn1,nn2</value>
   </property>
   <property>
     <name>fs.defaultFS</name>
-    <value>hdfs://{{ cluster_id }}</value>
+    <value>hdfs://{{ hdfs_cluster_id }}</value>
   </property>
   <property>
-    <name>dfs.namenode.rpc-address.{{ cluster_id }}.nn1</name>
+    <name>dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn1</name>
     <value>{{ namenode_1 }}:8020</value>
   </property>
   <property>
-    <name>dfs.namenode.rpc-address.{{ cluster_id }}.nn2</name>
+    <name>dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn2</name>
     <value>{{ namenode_2 }}:8020</value>
   </property>
   <property>
-     <name>dfs.namenode.http-address.{{ cluster_id }}.nn1</name>
+     <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1</name>
      <value>{{ namenode_1 }}:50070</value>
   </property>
   <property>
-     <name>dfs.namenode.http-address.{{ cluster_id }}.nn2</name>
+     <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2</name>
      <value>{{ namenode_2}}:50070</value>
   </property>
   <property>
-    <name>dfs.namenode.http-address.{{ cluster_id }}.nn1</name>
+    <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1</name>
     <value>{{ namenode_1 }}:9870</value>
   </property>
   <property>
-    <name>dfs.namenode.http-address.{{ cluster_id }}.nn2</name>
+    <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2</name>
     <value>{{ namenode_2 }}:9870</value>
   </property>
   <property>
@@ -118,4 +112,3 @@ HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config_template)
 write_file ${HDFS_CONFIG_TEMPLATE} "${HDFS_CONFIG}"
 write_file ${HDFS_CONFIG_TEMPLATE_CORE} "${HDFS_CONFIG_CORE}"
 write_file ${HDFS_CONFIG_TEMPLATE_MAPRED} "${HDFS_CONFIG_MAPRED}"
-write_file ${HDFS_CONFIG_DATANODES} "localhost"

tests/ansible-hadoop/roles/hdfs/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+  - name: Download Hadoop archive if not existent
+    get_url:
+      url: "{{ hadoop_archive_url }}"
+      dest: "{{ hadoop_path }}/{{ hadoop_archive_file }}"
+      mode: 0600
+      checksum: sha256:{{ hadoop_archive_file_checksum256 }}
+    run_once: true
+    delegate_to: localhost
+
+  - name: Extract archive
+    ansible.builtin.unarchive:
+      src: "{{ hadoop_path }}/{{ hadoop_archive_file }}"
+      dest: "{{ hadoop_path }}"
+
+  - name: Recursively remove directory
+    ansible.builtin.file:
+      path: "{{ hadoop_path }}/{{ hadoop_dir }}"
+      state: absent
+
+  - name: Rename dir
+    command: mv -f {{ hadoop_path }}/{{ hadoop_archive_dir }} {{ hadoop_path }}/{{ hadoop_dir }}
+
+  - name: Write config templates
+    ansible.builtin.command:
+      cmd: "roles/hdfs/files/hadoop-config.sh"
+    delegate_to: localhost
+
+  - name: Create core-site.xml
+    template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/core-site.xml mode=0700
+
+  - name: Create hdfs-site.xml
+    template: src={{ hadoop_config_path}}/hdfs-site.xml.j2 dest={{ hadoop_config_path}}/hdfs-site.xml mode=0700
+
+  - name: Create mapred-site.xml
+    template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/mapred-site.xml mode=0700

tests/hbase/tasks/main.yml

@@ -1,30 +0,0 @@
----
-
-  - name: Download HBase archive
-    get_url:
-      url: "{{ hbase_archive_url }}"
-      dest: "{{ script_path }}"
-      mode: 0600
-      checksum: sha256:{{ hbase_archive_file_checksum256 }}
-    run_once: true
-    delegate_to: localhost
-
-  - name: Copy archive
-    copy:
-      src: "{{ script_path }}/{{ hbase_archive }}"
-      dest: "/tmp"
-      remote_src: yes
-
-  - name: Extract archive
-    ansible.builtin.unarchive:
-      src: "/tmp/{{ hbase_archive }}"
-      dest: "/tmp"
-      remote_src: yes
-
-  - name: Write config template
-    ansible.builtin.command:
-      cmd: "{{ script_path }}/hbase/files/hbase-config.sh"
-    delegate_to: localhost
-
-  - name: Create hbase-site.xml
-    template: src={{ script_path }}/{{ hbase_config_path }}/hbase-site.xml.j2 dest={{ script_path }}/{{ hbase_config_path }}/hbase-site.xml mode=0700

tests/hdfs/defaults/main.yml

@@ -1,4 +0,0 @@
-hadoop_version: "2.10.0"
-hadoop_archive_file: "hadoop-{{ hadoop_version }}.tar.gz"
-hadoop_archive_file_checksum256: "273d5fa1d479d0bb96759b16cf4cbd6ba3e7f863a0778cbae55ab83417e961f0"
-hadoop_archive_url: "https://artfiles.org/apache.org/hadoop/common/hadoop-{{ hadoop_version }}}/{{ hadoop_archive_file }}"

tests/hdfs/tasks/main.yml

@@ -1,36 +0,0 @@
----
-
-  - name: Download Hadoop archive
-    get_url:
-      url: "{{ hadoop_archive_url }}"
-      dest: "{{ script_path }}"
-      mode: 0600
-      checksum: sha256:{{ hadoop_archive_file_checksum256 }}
-    run_once: true
-    delegate_to: localhost
-
-  - name: Copy archive
-    copy:
-      src: "{{ script_path }}/{{ hadoop_archive }}"
-      dest: "/tmp"
-      remote_src: yes
-
-  - name: Extract archive
-    ansible.builtin.unarchive:
-      src: "/tmp/{{ hadoop_archive }}"
-      dest: "/tmp"
-      remote_src: yes
-
-  - name: Write config template
-    ansible.builtin.command:
-      cmd: "{{ script_path }}/hdfs/files/hadoop-config.sh"
-    delegate_to: localhost
-
-  - name: Create core-site.xml
-    template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/core-site.xml mode=0700
-
-  - name: Create hdfs-site.xml
-    template: src={{ hdfs_config_path}}/hdfs-site.xml.j2 dest={{ hdfs_config_path}}/hdfs-site.xml mode=0700
-
-  - name: Create mapred-site.xml
-    template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/mapred-site.xml mode=0700

tests/test_hbase_exporter.sh

@@ -1,18 +1,10 @@
 #!/usr/bin/env bash
 
+SETUP_HBASE=false
+SETUP_HADOOP=true
 HBASE_TIME_STARTUP=15
 HBASE_EXPORTER_TIME_STARTUP=60
-HBASE_CMD="./bin/hbase-daemon.sh --config conf start master"
-HDFS_FORMAT=false
 HADOOP_CONFIG_DIR="hadoop/etc/hadoop"
-HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode"
-HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} datanode"
-HDFS_CMD_ZKFC="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} start zkfc"
-HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -format"
-HDFS_CMD_FORMAT_HA="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -initializeSharedEdits"
-
-SCRIPT_PATH=$(dirname "$0")
-source $SCRIPT_PATH/setup.sh
 
 setup_suite() {
     if [ "FreeBSD" = "$(uname)" ]; then
@@ -23,44 +15,37 @@ setup_suite() {
 
     export HADOOP_PREFIX="$(pwd)/hadoop"
 
-    # Setup HBase
+    cd ansible-hadoop/ || exit 1
-    if ! ansible-playbook -i inventory.yml \
-                          -e "hdfs_config_path=${HADOOP_PREFIX}/etc/hadoop" \
-                          -e "script_path=$(pwd)" \
-                          hbase-create.yml; then
 
-         printf "Failed to setup HBase to run test suite\n"
+    SCRIPT_PATH=$(dirname "$0")
-         exit 1
+    source $SCRIPT_PATH/ansible-hadoop/setup.sh
+
+    # Setup Hadoop
+    if [ true = "$SETUP_HADOOP" ]; then
+        if ! ansible-playbook -i inventory.yml \
+                              -e hadoop_path="/tmp" \
+                              -e hdfs_cluster_id="test-cluster" \
+                              hdfs-create.yml; then
+
+             printf "Failed to setup HDFS to run test suite\n"
+             exit 1
+        fi
     fi
 
-    # Setup HDFS
+    # Setup HBase
-    if ! ansible-playbook -i inventory.yml \
+    if [ true = "$SETUP_HBASE" ]; then
-                          -e "hdfs_config_path=$(pwd)/hadoop/etc/hadoop" \
+        if ! ansible-playbook -i inventory.yml \
-                          -e "script_path=$(pwd)" \
+                              -e hbase_path="/tmp" \
-                          hdfs-create.yml; then
+                              hbase-create.yml; then
 
-         printf "Failed to setup HDFS to run test suite\n"
+             printf "Failed to setup HBase to run test suite\n"
-         exit 1
+             exit 1
+        fi
     fi
 
     # Start hdfs
-    if [ true = "$HDFS_FORMAT" ]; then
-        printf "Formatting %s\n" "$1"
-        r=run $HDFS_CMD_FORMAT "HDFS FORMAT"
-        r=run $HDFS_CMD_FORMAT_HA "HDFS FORMAT HA"
-    fi
-
-    run "$HDFS_CMD_NAMENODE" "HDFS Namenode"
-    run "$HDFS_CMD_DATANODE" "HDFS Datanode"
 
     # Start HBase
-    cd hbase/ || exit
-    run "$HBASE_CMD" "HBASE"
-    if [[ "$r" == *"Stop it first."* ]]; then
-        printf "HBase is already running. Stop it manually first, then run script again"
-        exit 1
-    fi
-    sleep $HBASE_TIME_STARTUP
 
     # Start exporter
     run_exporter