initial commit

2019-01-22 13:10:16 +01:00 · 2019-01-22 13:10:16 +01:00 · ee4a13c926
commit ee4a13c926
13 changed files with 571 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,58 @@
+# ansible
+*.retry
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
+# .tfvars files are managed as part of configuration and so should be included in
+# version control.
+#
+# example.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
--- a/1-terraform.yml
+++ b/1-terraform.yml
@ -0,0 +1,73 @@
+---
+- hosts: terraform-master.e2m
+  remote_user: hybris
+
+##########################################
+###### DEFINE YOUR MACHINES HERE
+  vars:
+    kubernetes:
+      - hostname: k8s-master
+        ipv6: 2001:470:6d:22c:42::1
+        mac_address: 52:54:00:b2:52:86
+      - hostname: k8s-node-1
+        ipv6: 2001:470:6d:22c:42::10
+        mac_address: 52:54:00:52:b2:b2
+      - hostname: k8s-node-2
+        ipv6: 2001:470:6d:22c:42::20
+        mac_address: 52:54:00:52:2f:86
+      - hostname: k8s-node-3
+        ipv6: 2001:470:6d:22c:42::30
+        mac_address: 52:54:00:52:2f:b2
+      - hostname: k8s-node-4
+        ipv6: 2001:470:6d:22c:42::40
+        mac_address: 52:54:00:b2:2f:86
+######
+##########################################
+
+  tasks:
+
+  # - name: download qcow2 cloud image
+  #   uri:
+  #     url:
+  #     dest:
+
+  - name: create vm definitions
+    template: 
+      src: "roles/terraform/files/cloud-init.tf.j2"
+      dest: "/home/hybris/terraform/{{ item.hostname }}.tf"
+      owner: hybris
+      group: hybris
+    with_items: "{{ kubernetes }}"
+
+  - name: create cloud-init config
+    template: 
+      src: "roles/terraform/files/cloud-init.cfg.j2"
+      dest: "/home/hybris/terraform/{{ item.hostname }}.cloud_init.cfg"
+      owner: hybris
+      group: hybris
+    with_items: "{{ kubernetes }}"
+
+  - name: create libvirt_provider config
+    copy: 
+      src: roles/terraform/files/libvirt_provider.tf
+      dest: /home/hybris/terraform/libvirt_provider.tf
+
+  - name: delete statefile if it exists
+    file:
+      path: /home/hybris/terraform/terraform.tfstate
+      state: absent
+
+  - name: initialize terraform
+    shell: terraform init
+    args:
+      chdir: /home/hybris/terraform
+
+  - name: plan terraform
+    shell: terraform plan -out=cloud-init-plan
+    args:
+      chdir: /home/hybris/terraform
+
+  - name: apply terraform
+    shell: terraform apply "cloud-init-plan"
+    args:
+      chdir: /home/hybris/terraform
--- a/2-setup-kubernetes.yml
+++ b/2-setup-kubernetes.yml
@ -0,0 +1,12 @@
+---
+- hosts: kubernetes
+  remote_user: hybris
+  become: yes
+
+  roles:
+    - dns
+    - proxy
+    - network
+    - kubernetes
+
+# TODO: set ipv6 static so it will stick even on reboots
--- a/3-kubeadm-init.yml
+++ b/3-kubeadm-init.yml
@ -0,0 +1,36 @@
+---
+- hosts: k8s-nodes
+  remote_user: hybris
+  become: yes
+
+  tasks:
+    - name: initialize kubernetes master
+      shell: kubeadm init --apiserver-advertise-address=2001:470:6d:22c:42::1 --pod-network-cidr=2001:470:6d:22c:42::/80
+      delegate_to: k8s-master.e2m
+      run_once: yes
+
+    - shell: kubeadm token create --print-join-command
+      register: results
+
+    - debug:
+        var: results.stdout
+
+    - set_fact:
+        k8s_token: "{{ results.stdout | regex_search(regexp, '\\2') | first }}"
+      vars:
+        regexp: '([^\s]+\s){4}([^\s]+)'
+
+    - debug:
+        var: k8s_token
+
+    - set_fact:
+        k8s_ca_cert_hash: "{{ results.stdout | regex_search(regexp, '\\2') | first }}"
+      vars:
+        regexp: '([^\s]+\s){6}([^\s]+)'
+
+    - debug:
+        var: k8s_ca_cert_hash
+
+    - name: join nodes to cluster
+      shell: kubeadm join [2001:470:6d:22c:42::1]:6443 --token {{ k8s_token }} --discovery-token-ca-cert-hash sha256:{{ k8s_ca_cert_hash }}
+      delegate_to: k8s-nodes
--- a/inventories/hosts
+++ b/inventories/hosts
@ -0,0 +1,12 @@
+[terraform]
+terraform-master.e2m host_ipv6=2001:470:6d:22c:43::1
+
+[kubernetes]
+k8s-master.e2m host_ipv6=2001:470:6d:22c:42::1 hostname=k8s-master.e2m
+k8s-nodes
+
+[k8s-nodes]
+k8s-node-1.e2m host_ipv6=2001:470:6d:22c:42::10 host_hostname=k8s-node-1.e2m
+k8s-node-2.e2m host_ipv6=2001:470:6d:22c:42::20 host_hostname=k8s-node-2.e2m
+k8s-node-3.e2m host_ipv6=2001:470:6d:22c:42::30 host_hostname=k8s-node-3.e2m
+k8s-node-4.e2m host_ipv6=2001:470:6d:22c:42::40 host_hostname=k8s-node-4.e2m
--- a/roles/dns/files/resolv.conf
+++ b/roles/dns/files/resolv.conf
@ -0,0 +1,2 @@
+search local e2m
+nameserver 2001:470:6d:22c::1
--- a/roles/dns/tasks/main.yml
+++ b/roles/dns/tasks/main.yml
@ -0,0 +1,4 @@
+  - name: copy dns configuration
+    copy:
+      src: ../files/resolv.conf
+      dest: /etc/resolv.conf
--- a/roles/kubernetes/tasks/main.yml
+++ b/roles/kubernetes/tasks/main.yml
@ -0,0 +1,160 @@
+---
+########################
+#### FIREWALL TASKS ####
+########################
+
+# ## kubernetes requirements
+
+#   - name: enable port 6443/tcp
+#     firewalld:
+#       port: 6443/tcp
+#       permanent: yes
+#       state: enabled
+
+#   - name: enable port 10250/tcp
+#     firewalld:
+#       port: 10250/tcp
+#       permanent: yes
+#       state: enabled
+
+#   - name: enable port 6443/udp
+#     firewalld:
+#       port: 6443/udp
+#       permanent: yes
+#       state: enabled
+
+#   - name: enable port 10250/udp
+#     firewalld:
+#       port: 10250/udp
+#       permanent: yes
+#       state: enabled
+
+# ## reload firewalld after setting rules
+
+#   - name: reload firewalld
+#     shell: firewall-cmd --reload
+
+######################
+#### UPDATE TASKS ####
+######################
+
+  - name: upgrade all packages
+    yum:
+      name: '*'
+      state: latest
+
+######################
+#### KERNEL TASKS ####
+######################
+
+  - name: import elrepo gpg key
+    shell: rpm -httpproxy http://[2001:470:6d:22c::1]:3128 --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
+
+  - name: enable elrepo-release rpm
+    shell: rpm -httpproxy http://[2001:470:6d:22c::1]:3128 -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
+
+  # - name: Add repository
+  #   yum_repository:
+  #     name: elrepo-kernel
+  #     description: elrepo-release
+  #     baseurl: http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
+
+  - name: install mainline kernel
+    shell: yum --enablerepo=elrepo-kernel install kernel-ml -y
+
+  - name: set default kernel version in grub
+    lineinfile:
+      dest: /etc/default/grub
+      regexp: "^GRUB_DEFAULT"
+      line: "GRUB_DEFAULT=0"
+
+  - name: write grub config
+    shell: grub2-mkconfig -o /boot/grub2/grub.cfg
+
+####################
+#### MAIN TASKS ####
+####################
+
+  - name: permanently disable selinux
+    lineinfile:
+      dest: /etc/sysconfig/selinux
+      regexp: "^SELINUX="
+      line: "SELINUX=disabled"
+
+  - name: temporarily disable swap
+    shell: swapoff -a
+
+  - name: permanently disable swap
+    lineinfile:
+      dest: /etc/fstab
+      regexp: "^/dev/mapper/centos-swap"
+      line: "# /dev/mapper/centos-swap swap swap defaults 0 0"
+
+  - name: activate kernel module
+    shell: modprobe br_netfilter
+
+  - name: enable bridge-nf-call-iptables
+    sysctl:
+      name: net.bridge.bridge-nf-call-iptables
+      value: 1
+      sysctl_set: yes
+      state: present
+      reload: yes
+
+  - name: enable bridge-nf-call-ip6tables
+    sysctl: 
+      name: net.bridge.bridge-nf-call-ip6tables
+      value: 1
+      sysctl_set: yes
+      state: present
+      reload: yes
+
+  - name: enable ipv6 default forwarding
+    sysctl:
+      name: net.ipv6.conf.default.forwarding
+      value: 1
+      sysctl_set: yes
+      state: present
+      reload: yes
+
+  - name: add docker-ce yum repository
+    shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+  - name: copy kubernetes repo config
+    copy:
+      src: /Users/hybris/dev/k8s-ansible/files/kubernetes.repo
+      dest: /etc/yum.repos.d/kubernetes.repo
+
+  - name: install packages
+    yum:
+      name:
+      - yum-utils
+      - device-mapper-persistent-data
+      - lvm2
+      - docker-ce
+      - kubelet
+      - kubeadm
+      - kubectl
+      state: present
+
+  - name: set cgroup
+    lineinfile:
+      dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+      regexp: "^cgroup-driver="
+      line: "cgroup-driver=cgroupfs"
+
+  - name: force systemd to reread configs and restart service kubelet
+    systemd:
+      name: docker
+      enabled: yes
+      state: restarted
+
+  - name: force systemd to reread configs and restart service kubelet
+    systemd:
+      name: kubelet
+      enabled: yes
+      state: restarted
+      daemon_reload: yes
+
+  - name: reboot
+    reboot:
--- a/roles/proxy/tasks/main.yml
+++ b/roles/proxy/tasks/main.yml
@ -0,0 +1,69 @@
+---
+
+  - name: set proxy for yum
+    lineinfile:
+      dest: /etc/yum.conf
+      regexp: "^proxy="
+      line: "proxy=http://[2001:470:6d:22c::1]:3128"
+
+
+  - name: ensure wgetrc exists
+    file:
+      path: /etc/wgetrc
+      state: touch
+
+  - name: set proxy for wget
+    lineinfile:
+      dest: /etc/wgetrc
+      regexp: "{{ item.regexp }}"
+      line: "{{ item.line }}"
+    with_items:
+      - { regexp: '^http_proxy', line: 'http_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^https_proxy', line: 'https_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^ftp_proxy', line: 'ftp_proxy=http://[2001:470:6d:22c::1]:3128' }
+
+  - name: set proxy systemwide
+    lineinfile:
+      dest: /etc/environment
+      regexp: "{{ item.regexp }}"
+      line: "{{ item.line }}"
+    with_items:
+      - { regexp: '^export\ http_proxy', line: 'export http_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^export\ https_proxy', line: 'export https_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^export\ ftp_proxy', line: 'export ftp_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^export\ no_proxy', line: 'export no_proxy=localhost,::1,' }
+
+
+  - name: ensure proxy env file exists
+    file:
+      path: /etc/profile.d/webproxy.sh
+      state: touch
+      mode: "u=rw,g=r,o=r"
+
+  - name: set proxy environment variables
+    lineinfile:
+      dest: /etc/profile.d/webproxy.sh
+      regexp: "{{ item.regexp }}"
+      line: "{{ item.line }}"
+    with_items:
+      - { regexp: '^HTTP_PROXY', line: 'HTTP_PROXY=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^HTTPS_PROXY', line: 'HTTPS_PROXY=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^http_proxy', line: 'http_proxy=http://[2001:470:6d:22c::1]:3128' }
+      - { regexp: '^https_proxy', line: 'https_proxy=http://[2001:470:6d:22c::1]:3128' }
+
+
+  - name: ensure systemd conf folder exists
+    file:
+      path: /etc/systemd/system.conf.d
+      state: directory
+
+  - name: ensure systemd default env file exists
+    file:
+      path: /etc/systemd/system.conf.d/10-default-env.conf
+      state: touch
+
+  - name: set proxy for systemd
+    lineinfile:
+      dest: /etc/systemd/system.conf.d/10-default-env.conf
+      regexp: "^DefaultEnvironment=HTTP_PROXY"
+      line: "DefaultEnvironment=HTTP_PROXY=http://[2001:470:6d:22c::1]:3128"
--- a/roles/terraform/files/cloud-init.cfg.j2
+++ b/roles/terraform/files/cloud-init.cfg.j2
@ -0,0 +1,20 @@
+#cloud-config
+users:
+  - name: hybris
+    lock-passwd: false
+    passwd: $1$tG6Uv4$BPCIRF6RFuLrJ.lQO1GB8.
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    ssh_authorized_keys:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDXh0iMMtbTEhRSgIbgCunAuE+Q1iKycRVRBYWDCQ45xE8nHsg7K98fdv6LqQwkTqlojEHKp2/TPawk8V6w3MTgOHXi/WO9FyvhMjUcfdxTvny1z3Pj1j0NglKjyQ3t30snwOBYJiC2qDDtvRYNJs+Re/aSdVbygOHMf0UqCyP8DL7Tcj5sNPgnleEC7jE+YNbyDJo2KtNFXy3+AMbwa742D5Ygi5FqaBO090gVQfb3hbnZLw9CNSK7vjo7k5hJo8d1kcPDth1Vo12IDejL8Al3ArfGSy5IqYXs8Stj4YVRW+78AOpQZiInCaFybqjsGG+5tezrXHrxhjJPHuSCuXk9vxkcUk7aC5eSDqoCP7RTB+iQCPJ1ZvkVbaebExZGgHpD9t2Xo2bd/3BRagIxPTzKo9q4+qCCClhWR4iUSOr/YKAZBFDdoVWg2IMqbUHzvHX7TE7I8hbAdg6CpgwBoJm8OWmfjkDU6aYnF8z9l36VV7xjvvuYbOrhpCJ9LGFSfmG3wswo8OH4XyRSHd3y7qrSYIGujtymrzozXo2MrMU3Rj2bW5MIEZSxleIVje4PHWOfZwwn3hwvqM/nyknzfNwPCidTsP3Q2FwfBtZYo1cIXH41bzjs4Bxb8+KJqwTcerzB1pT4qsoPnoYW0fnZE7NHVNhfNx53LdctclFerHTLMw== hybris@sparks
+write_files:
+-   content: |
+        #!/bin/bash
+        # inet6_addr=$(ip a s | grep "inet6 2001:470:6d:22c:" | awk '{print substr($2,0)}')
+        sudo hostnamectl set-hostname {{ item.hostname }}
+        sudo ip -6 addr add {{ item.ipv6 }}/64 dev eth0
+        sudo chown -R hybris:hybris /home/hybris
+    path: /home/hybris/cloud-init.sh
+    owner: hybris:hybris
+    permissions: '0744'
+runcmd:
+ - [ '/home/hybris/cloud-init.sh' ]
--- a/roles/terraform/files/cloud-init.tf.j2
+++ b/roles/terraform/files/cloud-init.tf.j2
@ -0,0 +1,55 @@
+resource "libvirt_cloudinit_disk" "cloud-init-{{ item.hostname }}" {
+  name      = "cloud-init-{{ item.hostname }}.iso"
+  pool      = "kubernetes"
+  user_data = "${data.template_file.user-data-{{ item.hostname }}.rendered}"
+}
+
+data "template_file" "user-data-{{ item.hostname }}" {
+  template = "${file("${path.module}/{{ item.hostname }}.cloud_init.cfg")}"
+}
+
+resource "libvirt_volume" "centos-7-generic-{{ item.hostname }}" {
+  name     = "centos-7-generic-{{ item.hostname }}"
+  source   = "CentOS-7-x86_64-GenericCloud.qcow2"
+  pool     = "kubernetes"
+  format   = "qcow2"
+}
+
+resource "libvirt_domain" "domain-{{ item.hostname }}" {
+  name   = "{{ item.hostname }}"
+  memory = "1024"
+  vcpu   = 1
+
+  cloudinit = "${libvirt_cloudinit_disk.cloud-init-{{ item.hostname }}.id}"
+
+  console {
+    type        = "pty"
+    target_port = "0"
+    target_type = "serial"
+  }
+
+  console {
+      type        = "pty"
+      target_type = "virtio"
+      target_port = "1"
+  }
+
+  network_interface {
+    bridge = "home-lan"
+    mac    = "{{ item.mac_address }}"
+  }
+
+  boot_device {
+    dev = ["hd"]
+  }
+
+  disk {
+    volume_id = "${libvirt_volume.centos-7-generic-{{ item.hostname }}.id}"
+  }
+
+  graphics {
+    type        = "spice"
+    listen_type = "address"
+    autoport    = true
+  }
+}
--- a/roles/terraform/files/libvirt_provider.tf
+++ b/roles/terraform/files/libvirt_provider.tf
@ -0,0 +1,3 @@
+provider "libvirt" {
+    uri = "qemu+ssh://[2001:470:6d:22c:1::1]/system?socket=/var/run/libvirt/libvirt-sock"
+}
--- a/roles/terraform/tasks/main.yml
+++ b/roles/terraform/tasks/main.yml
@ -0,0 +1,67 @@
+---
+  - name: install packages
+    yum:
+      name:
+      - libvirt-devel
+      - git
+      - gcc
+      - unzip
+      state: present
+    become: yes
+
+  - name: download and install terraform 0.11.11 release
+    unarchive:
+      src: https://releases.hashicorp.com/terraform/0.11.11/terraform_0.11.11_linux_amd64.zip
+      dest: /usr/local/bin
+      remote_src: yes
+      become: yes
+
+  - name: download and install golang 1.11.4 release
+    unarchive:
+      src: https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz
+      dest: /usr/local
+      remote_src: yes
+      become: yes
+
+  - name: export path
+    lineinfile:
+      path: /etc/profile
+      regexp: '^export PATH=$PATH:/usr/local/go/bin'
+      line: 'export PATH=$PATH:/usr/local/go/bin'
+    become: yes
+
+  - name: go get terraform-provider-libvirt
+    shell: /usr/local/go/bin/go get github.com/dmacvicar/terraform-provider-libvirt
+
+  - name: go install terraform-provider-libvirt
+    shell: /usr/local/go/bin/go install
+    args:
+      chdir: /home/hybris/go/src/github.com/dmacvicar/terraform-provider-libvirt
+
+  - name: create terraform config directory
+    file:
+      path: /home/hybris/.terraform.d/
+      state: directory
+
+  - name: create terraform plugin directory
+    file:
+      path: /home/hybris/.terraform.d/plugins
+      state: directory
+
+  - name: install terraform-provider-libvirt
+    copy:
+      src: /home/hybris/go/bin/terraform-provider-libvirt
+      dest: /home/hybris/.terraform.d/plugins/terraform-provider-libvirt
+      mode: 0777
+      owner: hybris
+      remote_src: yes
+
+  - name: delete terraform directory
+    file:
+      path: /home/hybris/terraform
+      state: absent
+
+  - name: create terraform directory
+    file:
+      path: /home/hybris/terraform
+      state: directory