Add files

roles/hdfs/defaults/main.yml

@@ -0,0 +1,15 @@
+hadoop_version: "2.10.0"
+hadoop_dir: "hadoop"
+hadoop_archive_file: "hadoop-{{ hadoop_version }}.tar.gz"
+hadoop_archive_dir: "hadoop-{{ hadoop_version }}"
+hadoop_archive_file_checksum256: "131750c258368be4baff5d4a83b4de2cd119bda3774ed26d1d233b6fdf33f07f"
+hadoop_archive_url: "https://archive.apache.org/dist/hadoop/common/hadoop-{{ hadoop_version }}/{{ hadoop_archive_file }}"
+hadoop_config_path: "{{ hadoop_path }}/hadoop/etc/hadoop"
+hdfs_cmd_namenode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode"
+hdfs_cmd_datanode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs datanode"
+hdfs_cmd_journalnode: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs journalnode"
+hdfs_cmd_zkfc: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs start zkfc"
+hdfs_cmd_format: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode -format"
+hdfs_cmd_format_ha: "{{ hadoop_path }}/{{ hadoop_dir }}/bin/hdfs namenode -initializeSharedEdits"
+namenode_1: "namenode-1"
+namenode_2: "namenode_2"

roles/hdfs/files/hadoop-config.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+
+set -ueo pipefail
+
+HDFS_CONFIG_TEMPLATE="/tmp/hadoop/etc/hadoop/hdfs-site.xml.j2"
+HDFS_CONFIG_TEMPLATE_CORE="/tmp/hadoop/etc/hadoop/core-site.xml.j2"
+HDFS_CONFIG_TEMPLATE_MAPRED="/tmp/hadoop/etc/hadoop/mapred-site.xml.j2"
+
+SCRIPT_PATH=$(dirname "$0")
+source setup.sh
+
+create_hdfs_core_config_template() {
+    #printf "Writing HDFS core-site.xml config\n"
+    read -r -d '' CONFIG <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://{{ hdfs_cluster_id }}</value>
+  </property>
+  <property>
+    <name>dfs.journalnode.edits.dir</name>
+    <value>/.tmp/hadoop</value>
+  </property>
+</configuration>
+EOF
+    echo "$CONFIG"
+}
+
+create_hdfs_mapred_config_template() {
+    #printf "Writing HDFS mapred-site.xml config\n"
+    read -r -d '' CONFIG <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://{{ cluster_ha_id }}</value>
+  </property>
+</configuration>
+EOF
+    echo "$CONFIG"
+}
+
+create_hdfs_config_template() {
+    #printf "Writing HDFS hdfs-site.xml config\n"
+    read -r -d '' CONFIG <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>dfs.nameservices</name>
+    <value>{{ hdfs_cluster_id }}</value>
+  </property>
+  <property>
+    <name>dfs.ha.namenodes.{{ hdfs_cluster_id }}</name>
+    <value>nn1,nn2</value>
+  </property>
+  <property>
+    <name>fs.defaultFS</name>
+    <value>hdfs://{{ hdfs_cluster_id }}</value>
+  </property>
+  <property>
+    <name>dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn1</name>
+    <value>{{ namenode_1 }}:8020</value>
+  </property>
+  <property>
+    <name>dfs.namenode.rpc-address.{{ hdfs_cluster_id }}.nn2</name>
+    <value>{{ namenode_2 }}:8020</value>
+  </property>
+  <property>
+     <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1</name>
+     <value>{{ namenode_1 }}:50070</value>
+  </property>
+  <property>
+     <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2</name>
+     <value>{{ namenode_2}}:50070</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn1</name>
+    <value>{{ namenode_1 }}:9870</value>
+  </property>
+  <property>
+    <name>dfs.namenode.http-address.{{ hdfs_cluster_id }}.nn2</name>
+    <value>{{ namenode_2 }}:9870</value>
+  </property>
+  <property>
+    <name>dfs.namenode.name.dir</name>
+    <value>/tmp/hdfs/namenode</value>
+  </property>
+  <property>
+    <name>dfs.datanode.data.dir</name>
+    <value>/tmp/hdfs/datanode</value>
+  </property>
+  <property>
+    <name>dfs.namenode.shared.edits.dir</name>
+    <value>file:///tmp/hadoop</value>
+  </property>
+  <property>
+    <name>ha.zookeeper.quorum</name>
+    <value>127.0.0.1:2181</value>
+  </property>
+</configuration>
+EOF
+    echo "$CONFIG"
+}
+
+HDFS_CONFIG=$(create_hdfs_config_template)
+HDFS_CONFIG_CORE=$(create_hdfs_core_config_template)
+HDFS_CONFIG_MAPRED=$(create_hdfs_mapred_config_template)
+write_file ${HDFS_CONFIG_TEMPLATE} "${HDFS_CONFIG}"
+write_file ${HDFS_CONFIG_TEMPLATE_CORE} "${HDFS_CONFIG_CORE}"
+write_file ${HDFS_CONFIG_TEMPLATE_MAPRED} "${HDFS_CONFIG_MAPRED}"

roles/hdfs/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+  - name: Download Hadoop archive if not existent
+    get_url:
+      url: "{{ hadoop_archive_url }}"
+      dest: "{{ hadoop_path }}/{{ hadoop_archive_file }}"
+      mode: 0600
+      checksum: sha256:{{ hadoop_archive_file_checksum256 }}
+    run_once: true
+    delegate_to: localhost
+
+  - name: Extract archive
+    ansible.builtin.unarchive:
+      src: "{{ hadoop_path }}/{{ hadoop_archive_file }}"
+      dest: "{{ hadoop_path }}"
+
+  - name: Recursively remove directory
+    ansible.builtin.file:
+      path: "{{ hadoop_path }}/{{ hadoop_dir }}"
+      state: absent
+
+  - name: Rename dir
+    command: mv -f {{ hadoop_path }}/{{ hadoop_archive_dir }} {{ hadoop_path }}/{{ hadoop_dir }}
+
+  - name: Write config templates
+    ansible.builtin.command:
+      cmd: "roles/hdfs/files/hadoop-config.sh"
+    delegate_to: localhost
+
+  - name: Create core-site.xml
+    template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/core-site.xml mode=0700
+
+  - name: Create hdfs-site.xml
+    template: src={{ hadoop_config_path}}/hdfs-site.xml.j2 dest={{ hadoop_config_path}}/hdfs-site.xml mode=0700
+
+  - name: Create mapred-site.xml
+    template: src={{ hadoop_config_path}}/core-site.xml.j2 dest={{ hadoop_config_path}}/mapred-site.xml mode=0700