testsuite: Use ansible

2021-03-07 23:53:00 +01:00 · 2021-03-07 23:53:00 +01:00 · 4b7189dfde
commit 4b7189dfde
parent 59a7187b1b
6 changed files with 82 additions and 34 deletions
--- a/tests/hdfs-create.yml
+++ b/tests/hdfs-create.yml
@ -0,0 +1,6 @@
+---
+
+- hosts: namenodes
+  roles:
+    - hdfs
+
--- a/tests/hdfs-setup.sh
+++ b/tests/hdfs-setup.sh
@ -6,32 +6,36 @@ HADOOP_VERSION="2.10.1"
 HADOOP_FILE="hadoop-$HADOOP_VERSION.tar.gz"
 HADOOP_URL="https://artfiles.org/apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}"
 HADOOP_FILE_CKSUM="2460e02cd1f80dfed7a8981bbc934c095c0a341435118bec781fd835ec2ebdc5543a03d92d24f2ddeebdfe1c2c460065ba1d394ed9a73cbb2020b40a8d8b5e07"
-HDFS_CONFIG_FILE="hadoop/etc/hadoop/hdfs-site.xml"
-HDFS_CONFIG_FILE_CORE="hadoop/etc/hadoop/core-site.xml"
-HDFS_CONFIG_FILE_MAPRED="hadoop/etc/hadoop/mapred-site.xml"
+HDFS_CONFIG_TEMPLATE="hadoop/etc/hadoop/hdfs-site.xml"
+HDFS_CONFIG_TEMPLATE_CORE="hadoop/etc/hadoop/core-site.xml"
+HDFS_CONFIG_TEMPLATE_MAPRED="hadoop/etc/hadoop/mapred-site.xml"
 HDFS_CONFIG_DATANODES="localhost"
 HDFS_TEST_SUITE_EXECUTABLE="hadoop/bin/hdfs"

 source setup.sh

-declare -a DEPS=("java")
+declare -a DEPS=("java" "ansible-playbook")

-create_hdfs_core_config() {
+create_hdfs_core_config_template() {
    #printf "Writing HDFS core-site.xml config\n"
    read -r -d '' CONFIG <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration>
-   <property>
-      <name>fs.defaultFS</name>
-      <value>hdfs://$2</value>
-   </property>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://{{ cluster_id }}</value>
+  </property>
+  <property>
+    <name>dfs.journalnode.edits.dir</name>
+    <value>/.tmp/hadoop</value>
+  </property>
 </configuration>
 EOF
    echo "$CONFIG"
 }

-create_hdfs_mapred_config() {
+create_hdfs_mapred_config_template() {
    #printf "Writing HDFS mapred-site.xml config\n"
    read -r -d '' CONFIG <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
@ -39,14 +43,14 @@ create_hdfs_mapred_config() {
 <configuration>
  <property>
    <name>fs.defaultFS</name>
-    <value>hdfs://$1</value>
+    <value>hdfs://{{ cluster_ha_id }}</value>
  </property>
 </configuration>
 EOF
    echo "$CONFIG"
 }

-create_hdfs_config() {
+create_hdfs_config_template() {
    #printf "Writing HDFS hdfs-site.xml config\n"
    read -r -d '' CONFIG <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
@ -54,31 +58,39 @@ create_hdfs_config() {
 <configuration>
  <property>
    <name>dfs.nameservices</name>
-    <value>$2</value>
+    <value>{{ cluster_id }}</value>
  </property>
  <property>
-    <name>dfs.ha.namenodes.$2</name>
+    <name>dfs.ha.namenodes.{{ cluster_id }}</name>
    <value>nn1,nn2</value>
  </property>
  <property>
    <name>fs.defaultFS</name>
-    <value>hdfs://$2</value>
+    <value>hdfs://{{ cluster_id }}</value>
  </property>
  <property>
-    <name>dfs.namenode.rpc-address.$2.nn1</name>
-    <value>master-1:8020</value>
+    <name>dfs.namenode.rpc-address.{{ cluster_id }}.nn1</name>
+    <value>{{ namenode_1 }}:8020</value>
  </property>
  <property>
-    <name>dfs.namenode.rpc-address.$2.nn2</name>
-    <value>master-2:8020</value>
+    <name>dfs.namenode.rpc-address.{{ cluster_id }}.nn2</name>
+    <value>{{ namenode_2 }}:8020</value>
  </property>
  <property>
-     <name>dfs.namenode.http-address.$2.nn1</name>
-     <value>master-1:50070</value>
+     <name>dfs.namenode.http-address.{{ cluster_id }}.nn1</name>
+     <value>{{ namenode_1 }}:50070</value>
  </property>
  <property>
-     <name>dfs.namenode.http-address.$2.nn2</name>
-     <value>master-2:50070</value>
+     <name>dfs.namenode.http-address.{{ cluster_id }}.nn2</name>
+     <value>{{ namenode_2}}:50070</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address.{{ cluster_id }}.nn1</name>
+    <value>{{ namenode_1 }}:9870</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address.{{ cluster_id }}.nn2</name>
+    <value>{{ namenode_2 }}:9870</value>
  </property>
  <property>
    <name>dfs.namenode.name.dir</name>
@ -128,10 +140,10 @@ prepare_hadoop() {

 check_dependencies
 prepare_hadoop ${HADOOP_URL}
-HDFS_CONFIG=$(create_hdfs_config "127.0.0.1:8020" "test-cluster")
-HDFS_CONFIG_CORE=$(create_hdfs_core_config "127.0.0.1:8020" "test-cluster")
-HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config "127.0.0.1:8021")
-write_file ${HDFS_CONFIG_FILE} "${HDFS_CONFIG}"
-write_file ${HDFS_CONFIG_FILE_CORE} "${HDFS_CONFIG_CORE}"
-write_file ${HDFS_CONFIG_FILE_MAPRED} "${HDFS_CONFIG_MAPRED}"
+HDFS_CONFIG=$(create_hdfs_config_template)
+HDFS_CONFIG_CORE=$(create_hdfs_core_config_template)
+HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config_template)
+write_file ${HDFS_CONFIG_TEMPLATE} "${HDFS_CONFIG}"
+write_file ${HDFS_CONFIG_TEMPLATE_CORE} "${HDFS_CONFIG_CORE}"
+write_file ${HDFS_CONFIG_TEMPLATE_MAPRED} "${HDFS_CONFIG_MAPRED}"
 write_file ${HDFS_CONFIG_DATANODES} "localhost"
--- a/tests/hdfs/defaults/main.yml
+++ b/tests/hdfs/defaults/main.yml
--- a/tests/hdfs/tasks/main.yml
+++ b/tests/hdfs/tasks/main.yml
@ -0,0 +1,15 @@
+---
+
+  - name: Download and extract HDFS archive
+    ansible.builtin.shell:
+      cmd: hdfs-setup.sh
+      chdir: "{{ hadoop_prefix }}"
+
+  - name: Create core-site.xml
+    template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/core-site.xml mode=0700
+
+  - name: Create hdfs-site.xml
+    template: src={{ hdfs_config_path}}/hdfs-site.xml.j2 dest={{ hdfs_config_path}}/hdfs-site.xml mode=0700
+
+  - name: Create mapred-site.xml
+    template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/mapred-site.xml mode=0700
--- a/tests/inventory.yml
+++ b/tests/inventory.yml
@ -0,0 +1,7 @@
+all:
+  children:
+    namenodes:
+      vars:
+      hosts:
+        namenode-1:
+        namenode-2:
--- a/tests/test_hbase_exporter.sh
+++ b/tests/test_hbase_exporter.sh
@ -4,9 +4,12 @@ HBASE_TIME_STARTUP=15
 HBASE_EXPORTER_TIME_STARTUP=60
 HBASE_CMD="./bin/hbase-daemon.sh --config conf start master"
 HDFS_FORMAT=false
-HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode"
-HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config hadoop/etc/hadoop datanode"
-HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode -format"
+HADOOP_CONFIG_DIR="hadoop/etc/hadoop"
+HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode"
+HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} datanode"
+HDFS_CMD_ZKFC="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} start zkfc"
+HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -format"
+HDFS_CMD_FORMAT_HA="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -initializeSharedEdits"

 source setup.sh

@ -26,7 +29,11 @@ setup_suite() {
    fi

    # Setup HDFS
-    if ! ./hdfs-setup.sh; then
+    if ! ansible-playbook -i inventory.yml \
+                          -e "hdfs_config_path=$(pwd)/hadoop/etc/hadoop" \
+                          -e "hadoop_prefix=$(pwd)" \
+                          hdfs-create.yml; then
+
         printf "Failed to setup HDFS to run test suite\n"
         exit 1
    fi
@ -34,7 +41,8 @@ setup_suite() {
    # Start hdfs
    if [ true = "$HDFS_FORMAT" ]; then
        printf "Formatting %s\n" "$1"
-        r=run $HDFS_CMD_FORMAT "HDFS_FORMAT"
+        r=run $HDFS_CMD_FORMAT "HDFS FORMAT"
+        r=run $HDFS_CMD_FORMAT_HA "HDFS FORMAT HA"
    fi

    run "$HDFS_CMD_NAMENODE" "HDFS Namenode"