From 65c915a07b02d228569c935ce9aa32709a048b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Busse?= Date: Sat, 6 Mar 2021 22:37:01 +0100 Subject: [PATCH] Add tests, include HDFS --- tests/hbase-setup.sh | 70 ++++++++++++------------ tests/hdfs-setup.sh | 103 +++++++++++++++++++++++++++++++++++ tests/setup.sh | 47 ++++++++++++++++ tests/test_hbase_exporter.sh | 76 ++++++++++++++++++++++---- 4 files changed, 249 insertions(+), 47 deletions(-) create mode 100755 tests/hdfs-setup.sh create mode 100644 tests/setup.sh diff --git a/tests/hbase-setup.sh b/tests/hbase-setup.sh index 3374729..f86c0e9 100755 --- a/tests/hbase-setup.sh +++ b/tests/hbase-setup.sh @@ -2,46 +2,20 @@ set -ueo pipefail -HBASE_VERSION="2.4.1" -HBASE_FILE="hbase-${HBASE_VERSION}-bin.tar.gz" -HBASE_URL="https://downloads.apache.org/hbase/${HBASE_VERSION}/${HBASE_FILE}" +HBASE_VERSION="0.96.1.1" +HBASE_FILE="hbase-${HBASE_VERSION}-hadoop2-bin.tar.gz" +#HBASE_URL="https://downloads.apache.org/hbase/${HBASE_VERSION}/${HBASE_FILE}" +HBASE_URL="https://archive.apache.org/dist/hbase/hbase-${HBASE_VERSION}/${HBASE_FILE}" HBASE_FILE_CKSUM="5afb643c2391461619516624168e042b42a66e25217a3319552264c6af522e3a21a5212bfcba759b7b976794648ef13ee7b5a415f33cdb89bba43d40162aa685" -HBASE_CONFIG="hbase/conf/hbase-site.xml" +HBASE_CONFIG_FILE="hbase/conf/hbase-site.xml" HBASE_TEST_SUITE_EXECUTABLE="hbase/bin/hbase" declare -a DEPS=("java") -check_dependencies() { - for i in "${DEPS[@]}" - do - if [[ -z $(which "${i}") ]]; then - error "Could not find ${i}" - exit 1 - fi - done -} +source setup.sh -prepare_hbase() { - if ! [ -f "$HBASE_TEST_SUITE_EXECUTABLE" ]; then - if [ -f "$HBASE_FILE" ]; then - CKSUM="$(sha512 -q ${HBASE_FILE})" - if [ "$CKSUM" = "$HBASE_FILE_CKSUM" ]; then - printf "HBase archive exists\n" - fi - else - printf "Downloading %s\n" "$1" - curl -LO "${1}" - fi - - printf "Extracting HBase archive\n" - tar xfz ${HBASE_FILE} - mv -f hbase-${HBASE_VERSION} hbase/ - fi -} - -create_config() { - printf "Writing HBase config\n" - cat < "$2" +create_hbase_config() { + read -r -d '' CONFIG < @@ -59,8 +33,34 @@ create_config() { EOF + echo "$CONFIG" +} + +prepare_hbase() { + if ! [ -f "$HBASE_TEST_SUITE_EXECUTABLE" ]; then + if [ -f "$HBASE_FILE" ]; then + printf "HBase archive exists\n" + if compare_checksum $HBASE_FILE $HBASE_FILE_CKSUM; then + extract_archive $HBASE_FILE $HBASE_VERSION + mv -f hbase-"${VERSION}" hbase/ + else + printf "HBase archive has wrong checksum (%s)\n" "$1" + printf "Execute script again to redownload file\n" + exit 1 + fi + fi + + printf "Downloading %s\n" "$1" + curl -LO "${1}" + + if compare_checksum $HBASE_FILE $HBASE_FILE_CKSUM; then + extract_archive $HBASE_FILE $HBASE_VERSION + mv -f hbase-${HBASE_VERSION} hbase/ + fi + fi } check_dependencies prepare_hbase ${HBASE_URL} -create_config "/tmp" ${HBASE_CONFIG} +HBASE_CONFIG=$(create_hbase_config "/tmp") +write_file ${HBASE_CONFIG_FILE} "${HBASE_CONFIG}" diff --git a/tests/hdfs-setup.sh b/tests/hdfs-setup.sh new file mode 100755 index 0000000..b32298d --- /dev/null +++ b/tests/hdfs-setup.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +set -ueo pipefail + +HADOOP_VERSION="2.10.1" +HADOOP_FILE="hadoop-$HADOOP_VERSION.tar.gz" +HADOOP_URL="https://artfiles.org/apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}" +HADOOP_FILE_CKSUM="2460e02cd1f80dfed7a8981bbc934c095c0a341435118bec781fd835ec2ebdc5543a03d92d24f2ddeebdfe1c2c460065ba1d394ed9a73cbb2020b40a8d8b5e07" +HDFS_CONFIG_FILE="hadoop/etc/hadoop/hdfs-site.xml" +HDFS_CONFIG_FILE_CORE="hadoop/etc/hadoop/core-site.xml" +HDFS_CONFIG_FILE_MAPRED="hadoop/etc/hadoop/mapred-site.xml" +HDFS_TEST_SUITE_EXECUTABLE="hadoop/bin/hdfs" + +source setup.sh + +declare -a DEPS=("java") + +create_hdfs_core_config() { + #printf "Writing HDFS core-site.xml config\n" + read -r -d '' CONFIG < + + + + fs.defaultFS + hdfs://$1 + + +EOF + echo "$CONFIG" +} + +create_hdfs_mapred_config() { + #printf "Writing HDFS mapred-site.xml config\n" + read -r -d '' CONFIG < + + + + fs.defaultFS + hdfs://$1 + + +EOF + echo "$CONFIG" +} + +create_hdfs_config() { + #printf "Writing HDFS hdfs-site.xml config\n" + read -r -d '' CONFIG < + + + + fs.defaultFS + hdfs://$1 + + + dfs.namenode.name.dir + /tmp/hdfs/namenode + + + dfs.datanode.data.dir + /tmp/hdfs/datanode + + +EOF + echo "$CONFIG" +} + +prepare_hadoop() { + if ! [ -f "$HDFS_TEST_SUITE_EXECUTABLE" ]; then + printf "Setting up Hadoop\n" + if [ -f "$HADOOP_FILE" ]; then + printf "Hadoop archive exists\n" + if compare_checksum $HADOOP_FILE $HADOOP_FILE_CKSUM; then + extract_archive "$HADOOP_FILE" "$HADOOP_VERSION" + mv -f hadoop-$HADOOP_VERSION hadoop/ + return + else + printf "Hadoop archive has wrong checksum (%s)\n" "$1" + printf "Execute script again to redownload file\n" + exit 1 + fi + fi + + printf "Downloading %s\n" "$1" + curl -LO "${1}" + if compare_checksum $HADOOP_FILE $HADOOP_FILE_CKSUM; then + extract_archive "$HADOOP_FILE" "$HADOOP_VERSION" + mv -f hadoop-$HADOOP_VERSION hadoop/ + fi + fi +} + +check_dependencies +prepare_hadoop ${HADOOP_URL} +HDFS_CONFIG=$(create_hdfs_config "127.0.0.1:8020") +HDFS_CONFIG_CORE=$(create_hdfs_core_config "127.0.0.1:8020") +HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config "127.0.0.1:8021") +write_file ${HDFS_CONFIG_FILE} "${HDFS_CONFIG}" +write_file ${HDFS_CONFIG_FILE_CORE} "${HDFS_CONFIG_CORE}" +write_file ${HDFS_CONFIG_FILE_MAPRED} "${HDFS_CONFIG_MAPRED}" diff --git a/tests/setup.sh b/tests/setup.sh new file mode 100644 index 0000000..d40f7f0 --- /dev/null +++ b/tests/setup.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +check_dependencies() { + for i in "${DEPS[@]}" + do + if [[ -z $(which "${i}") ]]; then + error "Could not find ${i}" + exit 1 + fi + done +} + +extract_archive() { + printf "Extracting %s archive\n" "$1" + if ! tar xfz "${1}"; then + printf "Failed to extract archive: %s\n" "$1" + exit 1 + fi +} + +compare_checksum() { + local r + CKSUM=$(sha512 -q "${1}") + if ! [ "$CKSUM" = "$2" ]; then + printf "File has wrong checksum (%s)\n" "$1" + r=1 + else + r=0 + fi + echo "$r" +} + +write_file() { + printf "Writing %s\n" "$1" + printf "%s" "$2" > "$1" +} + +run() { + printf "Starting %s\n" "$2" + if $($1 > /dev/null 2>&1 &); then + printf "Started %s successfully\n" "$2" + else + printf "Failed to start %s\n" "$2" + exit 1 + fi + echo "$!" +} diff --git a/tests/test_hbase_exporter.sh b/tests/test_hbase_exporter.sh index 414f48d..a93aacf 100644 --- a/tests/test_hbase_exporter.sh +++ b/tests/test_hbase_exporter.sh @@ -2,32 +2,68 @@ HBASE_TIME_STARTUP=15 HBASE_EXPORTER_TIME_STARTUP=60 +HBASE_CMD="./bin/hbase-daemon.sh --config conf start master" +HDFS_FORMAT=false +HDFS_CMD="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode" +HDFS_CMD_FORMAT="./hadoop/bin/hdfs --config hadoop/etc/hadoop namenode -format" + +source setup.sh setup_suite() { export JAVA_HOME=${JAVA_HOME:-"/usr/local"} + export HADOOP_PREFIX="$(pwd)/hadoop" # Setup HBase - ./hbase-setup.sh + if ! ./hbase-setup.sh; then + printf "Failed to setup HBase to run test suite\n" + exit 1 + fi - # Run HBase - cd hbase || exit - printf "Starting HBase in pseudo-distributed mode\n" - ./bin/hbase-daemon.sh --config conf start master + # Setup HDFS + if ! ./hdfs-setup.sh; then + printf "Failed to setup HDFS to run test suite\n" + exit 1 + fi + + # Start hdfs + if [ true = "$HDFS_FORMAT" ]; then + printf "Formatting %s\n" "$1" + r=run $HDFS_CMD_FORMAT "HDFS_FORMAT" + fi + + run "$HDFS_CMD" "HDFS" + + # Start HBase + cd hbase/ || exit + run "$HBASE_CMD" "HBASE" + if [[ "$r" == *"Stop it first."* ]]; then + printf "HBase is already running. Stop it manually first, then run script again" + exit 1 + fi sleep $HBASE_TIME_STARTUP - # Run exporter + # Start exporter + run_exporter + printf "Waiting %ss to gather exporter values\n" ${HBASE_EXPORTER_TIME_STARTUP} + sleep $HBASE_EXPORTER_TIME_STARTUP +} + +run_exporter() { cd ../../ || exit printf "Starting hbase-exporter\n" ./hbase-exporter --zookeeper-server="${ZK_SERVER:-"127.0.0.1"}" \ --hbase-pseudo-distributed=True \ --hbase-table="foo" > /dev/null 2>&1 & PID=$! - printf "Waiting %ss to gather exporter values\n" ${HBASE_EXPORTER_TIME_STARTUP} - sleep $HBASE_EXPORTER_TIME_STARTUP +} + +test_hdfs_up() { + assert "curl -s http://127.0.0.1:50070 > /dev/null" "HDFS: Namenode ui down" + assert "curl -s http://127.0.0.1:8021 > /dev/null" "HDFS: IPC down" } test_hbase_running() { - assert "nc -n -w1 \"${1:-\"127.0.0.1\"}\" \"${2:-\"16010\"}\"" + assert "nc -n -w1 \"${1:-\"127.0.0.1\"}\" \"${2:-\"16010\"}\"" "HBase: Not running" } test_hbase_zk_running() { @@ -59,8 +95,8 @@ test_hbase_exporter_export_zk_connection_count() { test_hbase_exporter_export_zk_has_leader() { r=$(curl -s http://127.0.0.1:9010 | grep '^zookeeper_has_leader' | cut -d " " -f2) - assert_not_equals "0.0" "$r" "exporer: Zookeeper has no leader" - assert_not_equals "" "$r" "exporer: Zookeeper has no leader" + assert_not_equals "0.0" "$r" "exporter: Zookeeper has no leader" + assert_not_equals "" "$r" "exporter: Zookeeper has no leader" } test_hbase_exporter_export_regionserver_live() { @@ -75,8 +111,24 @@ test_hbase_exporter_export_regionserver_dead() { assert_not_equals "" "$r" "exporter: HBase - Dead regionservers" } +test_hbase_exporter_export_hdfs_datanodes_live() { + r=$(curl -s http://127.0.0.1:9010 | grep '^hdfs_datanodes_live' | cut -d " " -f2) + assert_not_equals "0.0" "$r" "exporter: HDFS - No data nodes" + assert_not_equals "" "$r" "exporter: HDFS - No data nodes" +} + +test_hbase_exporter_export_hdfs_datanodes_dead() { + r=$(curl -s http://127.0.0.1:9010 | grep '^hdfs_datanodes_dead' | cut -d " " -f2) + assert_equals "0.0" "$r" "exporter: HDFS - Dead data nodes" + assert_not_equals "" "$r" "exporter: HDFS - Dead data nodes" +} + teardown_suite() { printf "Stopping hbase-exporter (%s)\n" "$PID" - kill $PID + if ! kill $PID > /dev/null 2>&1; then + printf "Failed to send SIGTERM to %s\n" "$PID" + fi + + printf "Stopping HBase\n" ./tests/hbase/bin/hbase-daemon.sh stop master }