commit ee4a13c926d81cdde6917c7af01dd415ad2aee10 Author: hybris Date: Tue Jan 22 13:10:16 2019 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9e4d119 --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +# ansible +*.retry + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Ignore any .tfvars files that are generated automatically for each Terraform run. Most +# .tfvars files are managed as part of configuration and so should be included in +# version control. +# +# example.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + diff --git a/1-terraform.yml b/1-terraform.yml new file mode 100644 index 0000000..3f7be94 --- /dev/null +++ b/1-terraform.yml @@ -0,0 +1,73 @@ +--- +- hosts: terraform-master.e2m + remote_user: hybris + +########################################## +###### DEFINE YOUR MACHINES HERE + vars: + kubernetes: + - hostname: k8s-master + ipv6: 2001:470:6d:22c:42::1 + mac_address: 52:54:00:b2:52:86 + - hostname: k8s-node-1 + ipv6: 2001:470:6d:22c:42::10 + mac_address: 52:54:00:52:b2:b2 + - hostname: k8s-node-2 + ipv6: 2001:470:6d:22c:42::20 + mac_address: 52:54:00:52:2f:86 + - hostname: k8s-node-3 + ipv6: 2001:470:6d:22c:42::30 + mac_address: 52:54:00:52:2f:b2 + - hostname: k8s-node-4 + ipv6: 2001:470:6d:22c:42::40 + mac_address: 52:54:00:b2:2f:86 +###### +########################################## + + tasks: + + # - name: download qcow2 cloud image + # uri: + # url: + # dest: + + - name: create vm definitions + template: + src: "roles/terraform/files/cloud-init.tf.j2" + dest: "/home/hybris/terraform/{{ item.hostname }}.tf" + owner: hybris + group: hybris + with_items: "{{ kubernetes }}" + + - name: create cloud-init config + template: + src: "roles/terraform/files/cloud-init.cfg.j2" + dest: "/home/hybris/terraform/{{ item.hostname }}.cloud_init.cfg" + owner: hybris + group: hybris + with_items: "{{ kubernetes }}" + + - name: create libvirt_provider config + copy: + src: roles/terraform/files/libvirt_provider.tf + dest: /home/hybris/terraform/libvirt_provider.tf + + - name: delete statefile if it exists + file: + path: /home/hybris/terraform/terraform.tfstate + state: absent + + - name: initialize terraform + shell: terraform init + args: + chdir: /home/hybris/terraform + + - name: plan terraform + shell: terraform plan -out=cloud-init-plan + args: + chdir: /home/hybris/terraform + + - name: apply terraform + shell: terraform apply "cloud-init-plan" + args: + chdir: /home/hybris/terraform diff --git a/2-setup-kubernetes.yml b/2-setup-kubernetes.yml new file mode 100644 index 0000000..e762dc8 --- /dev/null +++ b/2-setup-kubernetes.yml @@ -0,0 +1,12 @@ +--- +- hosts: kubernetes + remote_user: hybris + become: yes + + roles: + - dns + - proxy + - network + - kubernetes + +# TODO: set ipv6 static so it will stick even on reboots diff --git a/3-kubeadm-init.yml b/3-kubeadm-init.yml new file mode 100644 index 0000000..cf0fc7a --- /dev/null +++ b/3-kubeadm-init.yml @@ -0,0 +1,36 @@ +--- +- hosts: k8s-nodes + remote_user: hybris + become: yes + + tasks: + - name: initialize kubernetes master + shell: kubeadm init --apiserver-advertise-address=2001:470:6d:22c:42::1 --pod-network-cidr=2001:470:6d:22c:42::/80 + delegate_to: k8s-master.e2m + run_once: yes + + - shell: kubeadm token create --print-join-command + register: results + + - debug: + var: results.stdout + + - set_fact: + k8s_token: "{{ results.stdout | regex_search(regexp, '\\2') | first }}" + vars: + regexp: '([^\s]+\s){4}([^\s]+)' + + - debug: + var: k8s_token + + - set_fact: + k8s_ca_cert_hash: "{{ results.stdout | regex_search(regexp, '\\2') | first }}" + vars: + regexp: '([^\s]+\s){6}([^\s]+)' + + - debug: + var: k8s_ca_cert_hash + + - name: join nodes to cluster + shell: kubeadm join [2001:470:6d:22c:42::1]:6443 --token {{ k8s_token }} --discovery-token-ca-cert-hash sha256:{{ k8s_ca_cert_hash }} + delegate_to: k8s-nodes \ No newline at end of file diff --git a/inventories/hosts b/inventories/hosts new file mode 100644 index 0000000..721d44c --- /dev/null +++ b/inventories/hosts @@ -0,0 +1,12 @@ +[terraform] +terraform-master.e2m host_ipv6=2001:470:6d:22c:43::1 + +[kubernetes] +k8s-master.e2m host_ipv6=2001:470:6d:22c:42::1 hostname=k8s-master.e2m +k8s-nodes + +[k8s-nodes] +k8s-node-1.e2m host_ipv6=2001:470:6d:22c:42::10 host_hostname=k8s-node-1.e2m +k8s-node-2.e2m host_ipv6=2001:470:6d:22c:42::20 host_hostname=k8s-node-2.e2m +k8s-node-3.e2m host_ipv6=2001:470:6d:22c:42::30 host_hostname=k8s-node-3.e2m +k8s-node-4.e2m host_ipv6=2001:470:6d:22c:42::40 host_hostname=k8s-node-4.e2m diff --git a/roles/dns/files/resolv.conf b/roles/dns/files/resolv.conf new file mode 100644 index 0000000..e8512c0 --- /dev/null +++ b/roles/dns/files/resolv.conf @@ -0,0 +1,2 @@ +search local e2m +nameserver 2001:470:6d:22c::1 diff --git a/roles/dns/tasks/main.yml b/roles/dns/tasks/main.yml new file mode 100644 index 0000000..a3c1f7a --- /dev/null +++ b/roles/dns/tasks/main.yml @@ -0,0 +1,4 @@ + - name: copy dns configuration + copy: + src: ../files/resolv.conf + dest: /etc/resolv.conf \ No newline at end of file diff --git a/roles/kubernetes/tasks/main.yml b/roles/kubernetes/tasks/main.yml new file mode 100644 index 0000000..a604c7f --- /dev/null +++ b/roles/kubernetes/tasks/main.yml @@ -0,0 +1,160 @@ +--- +######################## +#### FIREWALL TASKS #### +######################## + +# ## kubernetes requirements + +# - name: enable port 6443/tcp +# firewalld: +# port: 6443/tcp +# permanent: yes +# state: enabled + +# - name: enable port 10250/tcp +# firewalld: +# port: 10250/tcp +# permanent: yes +# state: enabled + +# - name: enable port 6443/udp +# firewalld: +# port: 6443/udp +# permanent: yes +# state: enabled + +# - name: enable port 10250/udp +# firewalld: +# port: 10250/udp +# permanent: yes +# state: enabled + +# ## reload firewalld after setting rules + +# - name: reload firewalld +# shell: firewall-cmd --reload + +###################### +#### UPDATE TASKS #### +###################### + + - name: upgrade all packages + yum: + name: '*' + state: latest + +###################### +#### KERNEL TASKS #### +###################### + + - name: import elrepo gpg key + shell: rpm -httpproxy http://[2001:470:6d:22c::1]:3128 --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org + + - name: enable elrepo-release rpm + shell: rpm -httpproxy http://[2001:470:6d:22c::1]:3128 -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm + + # - name: Add repository + # yum_repository: + # name: elrepo-kernel + # description: elrepo-release + # baseurl: http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm + + - name: install mainline kernel + shell: yum --enablerepo=elrepo-kernel install kernel-ml -y + + - name: set default kernel version in grub + lineinfile: + dest: /etc/default/grub + regexp: "^GRUB_DEFAULT" + line: "GRUB_DEFAULT=0" + + - name: write grub config + shell: grub2-mkconfig -o /boot/grub2/grub.cfg + +#################### +#### MAIN TASKS #### +#################### + + - name: permanently disable selinux + lineinfile: + dest: /etc/sysconfig/selinux + regexp: "^SELINUX=" + line: "SELINUX=disabled" + + - name: temporarily disable swap + shell: swapoff -a + + - name: permanently disable swap + lineinfile: + dest: /etc/fstab + regexp: "^/dev/mapper/centos-swap" + line: "# /dev/mapper/centos-swap swap swap defaults 0 0" + + - name: activate kernel module + shell: modprobe br_netfilter + + - name: enable bridge-nf-call-iptables + sysctl: + name: net.bridge.bridge-nf-call-iptables + value: 1 + sysctl_set: yes + state: present + reload: yes + + - name: enable bridge-nf-call-ip6tables + sysctl: + name: net.bridge.bridge-nf-call-ip6tables + value: 1 + sysctl_set: yes + state: present + reload: yes + + - name: enable ipv6 default forwarding + sysctl: + name: net.ipv6.conf.default.forwarding + value: 1 + sysctl_set: yes + state: present + reload: yes + + - name: add docker-ce yum repository + shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + + - name: copy kubernetes repo config + copy: + src: /Users/hybris/dev/k8s-ansible/files/kubernetes.repo + dest: /etc/yum.repos.d/kubernetes.repo + + - name: install packages + yum: + name: + - yum-utils + - device-mapper-persistent-data + - lvm2 + - docker-ce + - kubelet + - kubeadm + - kubectl + state: present + + - name: set cgroup + lineinfile: + dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + regexp: "^cgroup-driver=" + line: "cgroup-driver=cgroupfs" + + - name: force systemd to reread configs and restart service kubelet + systemd: + name: docker + enabled: yes + state: restarted + + - name: force systemd to reread configs and restart service kubelet + systemd: + name: kubelet + enabled: yes + state: restarted + daemon_reload: yes + + - name: reboot + reboot: \ No newline at end of file diff --git a/roles/proxy/tasks/main.yml b/roles/proxy/tasks/main.yml new file mode 100644 index 0000000..f4d445b --- /dev/null +++ b/roles/proxy/tasks/main.yml @@ -0,0 +1,69 @@ +--- + + - name: set proxy for yum + lineinfile: + dest: /etc/yum.conf + regexp: "^proxy=" + line: "proxy=http://[2001:470:6d:22c::1]:3128" + + + - name: ensure wgetrc exists + file: + path: /etc/wgetrc + state: touch + + - name: set proxy for wget + lineinfile: + dest: /etc/wgetrc + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: '^http_proxy', line: 'http_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^https_proxy', line: 'https_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^ftp_proxy', line: 'ftp_proxy=http://[2001:470:6d:22c::1]:3128' } + + - name: set proxy systemwide + lineinfile: + dest: /etc/environment + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: '^export\ http_proxy', line: 'export http_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^export\ https_proxy', line: 'export https_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^export\ ftp_proxy', line: 'export ftp_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^export\ no_proxy', line: 'export no_proxy=localhost,::1,' } + + + - name: ensure proxy env file exists + file: + path: /etc/profile.d/webproxy.sh + state: touch + mode: "u=rw,g=r,o=r" + + - name: set proxy environment variables + lineinfile: + dest: /etc/profile.d/webproxy.sh + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + with_items: + - { regexp: '^HTTP_PROXY', line: 'HTTP_PROXY=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^HTTPS_PROXY', line: 'HTTPS_PROXY=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^http_proxy', line: 'http_proxy=http://[2001:470:6d:22c::1]:3128' } + - { regexp: '^https_proxy', line: 'https_proxy=http://[2001:470:6d:22c::1]:3128' } + + + - name: ensure systemd conf folder exists + file: + path: /etc/systemd/system.conf.d + state: directory + + - name: ensure systemd default env file exists + file: + path: /etc/systemd/system.conf.d/10-default-env.conf + state: touch + + - name: set proxy for systemd + lineinfile: + dest: /etc/systemd/system.conf.d/10-default-env.conf + regexp: "^DefaultEnvironment=HTTP_PROXY" + line: "DefaultEnvironment=HTTP_PROXY=http://[2001:470:6d:22c::1]:3128" \ No newline at end of file diff --git a/roles/terraform/files/cloud-init.cfg.j2 b/roles/terraform/files/cloud-init.cfg.j2 new file mode 100644 index 0000000..db61c36 --- /dev/null +++ b/roles/terraform/files/cloud-init.cfg.j2 @@ -0,0 +1,20 @@ +#cloud-config +users: + - name: hybris + lock-passwd: false + passwd: $1$tG6Uv4$BPCIRF6RFuLrJ.lQO1GB8. + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXh0iMMtbTEhRSgIbgCunAuE+Q1iKycRVRBYWDCQ45xE8nHsg7K98fdv6LqQwkTqlojEHKp2/TPawk8V6w3MTgOHXi/WO9FyvhMjUcfdxTvny1z3Pj1j0NglKjyQ3t30snwOBYJiC2qDDtvRYNJs+Re/aSdVbygOHMf0UqCyP8DL7Tcj5sNPgnleEC7jE+YNbyDJo2KtNFXy3+AMbwa742D5Ygi5FqaBO090gVQfb3hbnZLw9CNSK7vjo7k5hJo8d1kcPDth1Vo12IDejL8Al3ArfGSy5IqYXs8Stj4YVRW+78AOpQZiInCaFybqjsGG+5tezrXHrxhjJPHuSCuXk9vxkcUk7aC5eSDqoCP7RTB+iQCPJ1ZvkVbaebExZGgHpD9t2Xo2bd/3BRagIxPTzKo9q4+qCCClhWR4iUSOr/YKAZBFDdoVWg2IMqbUHzvHX7TE7I8hbAdg6CpgwBoJm8OWmfjkDU6aYnF8z9l36VV7xjvvuYbOrhpCJ9LGFSfmG3wswo8OH4XyRSHd3y7qrSYIGujtymrzozXo2MrMU3Rj2bW5MIEZSxleIVje4PHWOfZwwn3hwvqM/nyknzfNwPCidTsP3Q2FwfBtZYo1cIXH41bzjs4Bxb8+KJqwTcerzB1pT4qsoPnoYW0fnZE7NHVNhfNx53LdctclFerHTLMw== hybris@sparks +write_files: +- content: | + #!/bin/bash + # inet6_addr=$(ip a s | grep "inet6 2001:470:6d:22c:" | awk '{print substr($2,0)}') + sudo hostnamectl set-hostname {{ item.hostname }} + sudo ip -6 addr add {{ item.ipv6 }}/64 dev eth0 + sudo chown -R hybris:hybris /home/hybris + path: /home/hybris/cloud-init.sh + owner: hybris:hybris + permissions: '0744' +runcmd: + - [ '/home/hybris/cloud-init.sh' ] diff --git a/roles/terraform/files/cloud-init.tf.j2 b/roles/terraform/files/cloud-init.tf.j2 new file mode 100644 index 0000000..95e8d1f --- /dev/null +++ b/roles/terraform/files/cloud-init.tf.j2 @@ -0,0 +1,55 @@ +resource "libvirt_cloudinit_disk" "cloud-init-{{ item.hostname }}" { + name = "cloud-init-{{ item.hostname }}.iso" + pool = "kubernetes" + user_data = "${data.template_file.user-data-{{ item.hostname }}.rendered}" +} + +data "template_file" "user-data-{{ item.hostname }}" { + template = "${file("${path.module}/{{ item.hostname }}.cloud_init.cfg")}" +} + +resource "libvirt_volume" "centos-7-generic-{{ item.hostname }}" { + name = "centos-7-generic-{{ item.hostname }}" + source = "CentOS-7-x86_64-GenericCloud.qcow2" + pool = "kubernetes" + format = "qcow2" +} + +resource "libvirt_domain" "domain-{{ item.hostname }}" { + name = "{{ item.hostname }}" + memory = "1024" + vcpu = 1 + + cloudinit = "${libvirt_cloudinit_disk.cloud-init-{{ item.hostname }}.id}" + + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + + network_interface { + bridge = "home-lan" + mac = "{{ item.mac_address }}" + } + + boot_device { + dev = ["hd"] + } + + disk { + volume_id = "${libvirt_volume.centos-7-generic-{{ item.hostname }}.id}" + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} \ No newline at end of file diff --git a/roles/terraform/files/libvirt_provider.tf b/roles/terraform/files/libvirt_provider.tf new file mode 100644 index 0000000..6e71b7c --- /dev/null +++ b/roles/terraform/files/libvirt_provider.tf @@ -0,0 +1,3 @@ +provider "libvirt" { + uri = "qemu+ssh://[2001:470:6d:22c:1::1]/system?socket=/var/run/libvirt/libvirt-sock" +} \ No newline at end of file diff --git a/roles/terraform/tasks/main.yml b/roles/terraform/tasks/main.yml new file mode 100644 index 0000000..6f57123 --- /dev/null +++ b/roles/terraform/tasks/main.yml @@ -0,0 +1,67 @@ +--- + - name: install packages + yum: + name: + - libvirt-devel + - git + - gcc + - unzip + state: present + become: yes + + - name: download and install terraform 0.11.11 release + unarchive: + src: https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip + dest: /usr/local/bin + remote_src: yes + become: yes + + - name: download and install golang 1.11.4 release + unarchive: + src: https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz + dest: /usr/local + remote_src: yes + become: yes + + - name: export path + lineinfile: + path: /etc/profile + regexp: '^export PATH=$PATH:/usr/local/go/bin' + line: 'export PATH=$PATH:/usr/local/go/bin' + become: yes + + - name: go get terraform-provider-libvirt + shell: /usr/local/go/bin/go get github.com/dmacvicar/terraform-provider-libvirt + + - name: go install terraform-provider-libvirt + shell: /usr/local/go/bin/go install + args: + chdir: /home/hybris/go/src/github.com/dmacvicar/terraform-provider-libvirt + + - name: create terraform config directory + file: + path: /home/hybris/.terraform.d/ + state: directory + + - name: create terraform plugin directory + file: + path: /home/hybris/.terraform.d/plugins + state: directory + + - name: install terraform-provider-libvirt + copy: + src: /home/hybris/go/bin/terraform-provider-libvirt + dest: /home/hybris/.terraform.d/plugins/terraform-provider-libvirt + mode: 0777 + owner: hybris + remote_src: yes + + - name: delete terraform directory + file: + path: /home/hybris/terraform + state: absent + + - name: create terraform directory + file: + path: /home/hybris/terraform + state: directory