diff --git a/tests/hdfs-create.yml b/tests/hdfs-create.yml new file mode 100644 index 0000000..917bad1 --- /dev/null +++ b/tests/hdfs-create.yml @@ -0,0 +1,6 @@ +--- + +- hosts: namenodes + roles: + - hdfs + diff --git a/tests/hdfs-setup.sh b/tests/hdfs-setup.sh index 253c28e..451907b 100755 --- a/tests/hdfs-setup.sh +++ b/tests/hdfs-setup.sh @@ -6,32 +6,36 @@ HADOOP_VERSION="2.10.1" HADOOP_FILE="hadoop-$HADOOP_VERSION.tar.gz" HADOOP_URL="https://artfiles.org/apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}" HADOOP_FILE_CKSUM="2460e02cd1f80dfed7a8981bbc934c095c0a341435118bec781fd835ec2ebdc5543a03d92d24f2ddeebdfe1c2c460065ba1d394ed9a73cbb2020b40a8d8b5e07" -HDFS_CONFIG_FILE="hadoop/etc/hadoop/hdfs-site.xml" -HDFS_CONFIG_FILE_CORE="hadoop/etc/hadoop/core-site.xml" -HDFS_CONFIG_FILE_MAPRED="hadoop/etc/hadoop/mapred-site.xml" +HDFS_CONFIG_TEMPLATE="hadoop/etc/hadoop/hdfs-site.xml" +HDFS_CONFIG_TEMPLATE_CORE="hadoop/etc/hadoop/core-site.xml" +HDFS_CONFIG_TEMPLATE_MAPRED="hadoop/etc/hadoop/mapred-site.xml" HDFS_CONFIG_DATANODES="localhost" HDFS_TEST_SUITE_EXECUTABLE="hadoop/bin/hdfs" source setup.sh -declare -a DEPS=("java") +declare -a DEPS=("java" "ansible-playbook") -create_hdfs_core_config() { +create_hdfs_core_config_template() { #printf "Writing HDFS core-site.xml config\n" read -r -d '' CONFIG < - - fs.defaultFS - hdfs://$2 - + + fs.defaultFS + hdfs://{{ cluster_id }} + + + dfs.journalnode.edits.dir + /.tmp/hadoop + EOF echo "$CONFIG" } -create_hdfs_mapred_config() { +create_hdfs_mapred_config_template() { #printf "Writing HDFS mapred-site.xml config\n" read -r -d '' CONFIG < @@ -39,14 +43,14 @@ create_hdfs_mapred_config() { fs.defaultFS - hdfs://$1 + hdfs://{{ cluster_ha_id }} EOF echo "$CONFIG" } -create_hdfs_config() { +create_hdfs_config_template() { #printf "Writing HDFS hdfs-site.xml config\n" read -r -d '' CONFIG < @@ -54,31 +58,39 @@ create_hdfs_config() { dfs.nameservices - $2 + {{ cluster_id }} - dfs.ha.namenodes.$2 + dfs.ha.namenodes.{{ cluster_id }} nn1,nn2 fs.defaultFS - hdfs://$2 + hdfs://{{ cluster_id }} - dfs.namenode.rpc-address.$2.nn1 - master-1:8020 + dfs.namenode.rpc-address.{{ cluster_id }}.nn1 + {{ namenode_1 }}:8020 - dfs.namenode.rpc-address.$2.nn2 - master-2:8020 + dfs.namenode.rpc-address.{{ cluster_id }}.nn2 + {{ namenode_2 }}:8020 - dfs.namenode.http-address.$2.nn1 - master-1:50070 + dfs.namenode.http-address.{{ cluster_id }}.nn1 + {{ namenode_1 }}:50070 - dfs.namenode.http-address.$2.nn2 - master-2:50070 + dfs.namenode.http-address.{{ cluster_id }}.nn2 + {{ namenode_2}}:50070 + + + dfs.namenode.http-address.{{ cluster_id }}.nn1 + {{ namenode_1 }}:9870 + + + dfs.namenode.http-address.{{ cluster_id }}.nn2 + {{ namenode_2 }}:9870 dfs.namenode.name.dir @@ -128,10 +140,10 @@ prepare_hadoop() { check_dependencies prepare_hadoop ${HADOOP_URL} -HDFS_CONFIG=$(create_hdfs_config "127.0.0.1:8020" "test-cluster") -HDFS_CONFIG_CORE=$(create_hdfs_core_config "127.0.0.1:8020" "test-cluster") -HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config "127.0.0.1:8021") -write_file ${HDFS_CONFIG_FILE} "${HDFS_CONFIG}" -write_file ${HDFS_CONFIG_FILE_CORE} "${HDFS_CONFIG_CORE}" -write_file ${HDFS_CONFIG_FILE_MAPRED} "${HDFS_CONFIG_MAPRED}" +HDFS_CONFIG=$(create_hdfs_config_template) +HDFS_CONFIG_CORE=$(create_hdfs_core_config_template) +HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config_template) +write_file ${HDFS_CONFIG_TEMPLATE} "${HDFS_CONFIG}" +write_file ${HDFS_CONFIG_TEMPLATE_CORE} "${HDFS_CONFIG_CORE}" +write_file ${HDFS_CONFIG_TEMPLATE_MAPRED} "${HDFS_CONFIG_MAPRED}" write_file ${HDFS_CONFIG_DATANODES} "localhost" diff --git a/tests/hdfs/defaults/main.yml b/tests/hdfs/defaults/main.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/hdfs/tasks/main.yml b/tests/hdfs/tasks/main.yml new file mode 100644 index 0000000..266971f --- /dev/null +++ b/tests/hdfs/tasks/main.yml @@ -0,0 +1,15 @@ +--- + + - name: Download and extract HDFS archive + ansible.builtin.shell: + cmd: hdfs-setup.sh + chdir: "{{ hadoop_prefix }}" + + - name: Create core-site.xml + template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/core-site.xml mode=0700 + + - name: Create hdfs-site.xml + template: src={{ hdfs_config_path}}/hdfs-site.xml.j2 dest={{ hdfs_config_path}}/hdfs-site.xml mode=0700 + + - name: Create mapred-site.xml + template: src={{ hdfs_config_path}}/core-site.xml.j2 dest={{ hdfs_config_path}}/mapred-site.xml mode=0700 diff --git a/tests/inventory.yml b/tests/inventory.yml new file mode 100644 index 0000000..12068f0 --- /dev/null +++ b/tests/inventory.yml @@ -0,0 +1,7 @@ +all: + children: + namenodes: + vars: + hosts: + namenode-1: + namenode-2: diff --git a/tests/test_hbase_exporter.sh b/tests/test_hbase_exporter.sh index 4847caf..d93eeed 100644 --- a/tests/test_hbase_exporter.sh +++ b/tests/test_hbase_exporter.sh @@ -4,9 +4,12 @@ HBASE_TIME_STARTUP=15 HBASE_EXPORTER_TIME_STARTUP=60 HBASE_CMD="./bin/hbase-daemon.sh --config conf start master" HDFS_FORMAT=false -HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode" -HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config hadoop/etc/hadoop datanode" -HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode -format" +HADOOP_CONFIG_DIR="hadoop/etc/hadoop" +HDFS_CMD_NAMENODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode" +HDFS_CMD_DATANODE="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} datanode" +HDFS_CMD_ZKFC="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} start zkfc" +HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -format" +HDFS_CMD_FORMAT_HA="./hadoop/bin/hdfs --config ${HADOOP_CONFIG_DIR} namenode -initializeSharedEdits" source setup.sh @@ -26,7 +29,11 @@ setup_suite() { fi # Setup HDFS - if ! ./hdfs-setup.sh; then + if ! ansible-playbook -i inventory.yml \ + -e "hdfs_config_path=$(pwd)/hadoop/etc/hadoop" \ + -e "hadoop_prefix=$(pwd)" \ + hdfs-create.yml; then + printf "Failed to setup HDFS to run test suite\n" exit 1 fi @@ -34,7 +41,8 @@ setup_suite() { # Start hdfs if [ true = "$HDFS_FORMAT" ]; then printf "Formatting %s\n" "$1" - r=run $HDFS_CMD_FORMAT "HDFS_FORMAT" + r=run $HDFS_CMD_FORMAT "HDFS FORMAT" + r=run $HDFS_CMD_FORMAT_HA "HDFS FORMAT HA" fi run "$HDFS_CMD_NAMENODE" "HDFS Namenode"