k8s.tftpl

${file("${path_module}/userdata/generic")}
echo "starting k8s postinstall"

# Note: this is not very secure to use this key because:
# - it is on github without any protection
# - it does not have any password
cat <<EOF >/root/.ssh/authorized_keys
${file("${path_module}/../ansible/files/zob.pub")}
EOF

# Configure the network interfaces
cat <<'EOF' >/etc/systemd/network/10-octavia.link
[Match]
PermanentMACAddress=${mac_octavia}
[Link]
Name=octavia
EOF

cat <<'EOF' >/etc/systemd/network/10-octavia.network
[Match]
PermanentMACAddress=${mac_octavia}
[Network]
Address=172.21.0.5/16
EOF

systemctl restart systemd-udev-trigger.service
systemctl restart systemd-networkd.service

# Tune a little bit the number of open files
# This is useful for stern
cat <<'EOF' >/etc/sysctl.d/local.conf
fs.inotify.max_user_watches=999999
fs.inotify.max_user_instances=999999
fs.inotify.max_queued_events=999999
EOF

# Apply this new sysctl config
sysctl -p /etc/sysctl.d/local.conf

# Add useful aliases - not useful everywhere but at least we have them
cat << 'EOF' > /etc/profile.d/aliases.sh
alias k=kubectl
alias l=stern
alias os=openstack
alias frep-rabbit='frep /root/bootstrap-openstack-k8s/k8s/rabbit.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-keystone='frep /root/bootstrap-openstack-k8s/k8s/keystone.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-glance='frep /root/bootstrap-openstack-k8s/k8s/glance.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-placement='frep /root/bootstrap-openstack-k8s/k8s/placement.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-neutron='frep /root/bootstrap-openstack-k8s/k8s/neutron.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-nova='frep /root/bootstrap-openstack-k8s/k8s/nova.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-skyline='frep /root/bootstrap-openstack-k8s/k8s/skyline.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-mistral='frep /root/bootstrap-openstack-k8s/k8s/mistral.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-octavia='frep /root/bootstrap-openstack-k8s/k8s/octavia.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-redis='frep /root/bootstrap-openstack-k8s/k8s/redis.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-barbican='frep /root/bootstrap-openstack-k8s/k8s/barbican.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-loki='frep /root/bootstrap-openstack-k8s/k8s/loki.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
alias frep-garage='frep /root/bootstrap-openstack-k8s/k8s/garage.yaml.in:- --load /root/bootstrap-openstack-k8s/config/config.yaml | kubectl apply -f -'
EOF

# We may need docker to build images
# Add Docker's official GPG key:
apt-get update
apt-get install ca-certificates curl
install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

git clone https://github.com/arnaudmorin/bootstrap-openstack-k8s.git /root/bootstrap-openstack-k8s -b ${os_version}
git clone https://github.com/arnaudmorin/openstack-docker.git /root/openstack-docker -b ${os_version}

# Overwrite the zob key if this was changed locally
cat <<EOF >/root/bootstrap-openstack-k8s/ansible/files/zob.pub
${file("${path_module}/../ansible/files/zob.pub")}
EOF

cat <<EOF >/root/bootstrap-openstack-k8s/ansible/files/zob
${file("${path_module}/../ansible/files/zob")}
EOF

# Setup SSH config for easy access to computes and networks node
chmod 600 /root/bootstrap-openstack-k8s/ansible/files/zob

mkdir -p /root/.ssh/config.d/
echo "Include config.d/*" > /root/.ssh/config
tee /root/.ssh/config.d/os.conf << EOF
Match host="compute-*,network-*"
      User root
      StrictHostKeyChecking no
      UserKnownHostsFile /dev/null
      IdentityFile /root/bootstrap-openstack-k8s/ansible/files/zob
EOF

# Install k3s
curl -sfL https://get.k3s.io | sh -

kubectl get all

# Enable kubectl completion
kubectl completion bash > /etc/bash_completion.d/kubectl
echo 'complete -F __start_kubectl k' > /etc/profile.d/k.sh

# Install k9s
curl -sS https://webi.sh/k9s | HOME=/root sh

# Install frep
curl -fSL https://github.com/subchen/frep/releases/download/v1.3.12/frep-1.3.12-linux-amd64 -o /usr/local/bin/frep
chmod +x /usr/local/bin/frep

# Install osconsole (apache2-utils provides htpasswd for the garage-webui login)
apt-get install -y jq apache2-utils unzip
curl -fSL https://github.com/arnaudmorin/osconsole/releases/download/0.0.3/osconsole -o /usr/local/bin/osconsole
chmod +x /usr/local/bin/osconsole

# Install aws cli (v2) in /opt/aws with a symlink in /usr/local/bin (like openstack)
curl -fSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
unzip -q /tmp/awscliv2.zip -d /tmp
/tmp/aws/install --install-dir /opt/aws --bin-dir /usr/local/bin
rm -rf /tmp/awscliv2.zip /tmp/aws
# Enable aws cli bash completion (aws_completer is symlinked into /usr/local/bin).
# 's3' is a function (not an alias) so completion works: the wrapper rewrites the
# line to "aws s3 ..." and proxies it to aws_completer.
cat << 'EOF' > /etc/profile.d/aws_completion.sh
complete -C aws_completer aws

s3() { aws s3 "$@"; }
_s3_complete() {
    local line="aws s3 $${COMP_LINE#s3}"
    COMPREPLY=( $(COMP_LINE="$line" COMP_POINT="$${#line}" aws_completer) )
}
complete -F _s3_complete s3
EOF

cd /root/bootstrap-openstack-k8s
cp config/config.yaml.sample config/config.yaml
ip=$(hostname -I | awk '{print $1}')
sed -i -r "s/somewhere.net/$${ip}.xip.opensteak.fr/" config/config.yaml
sed -i -r "s/changeme/${password}/" config/config.yaml
sed -i -r "s/os_version/${os_version}/" config/config.yaml
# garage-webui login: bcrypt "admin:hash" (use | as sed delimiter, the hash contains /)
webui_auth=$(htpasswd -nbBC 10 admin "${password}")
sed -i "s|webui_auth_hash|$${webui_auth}|" config/config.yaml

# Stakater reloader
# This is not mandatory, but useful to auto-restart deployments / statefulset when a configmap change
kubectl apply -f https://raw.githubusercontent.com/stakater/Reloader/v1.4.8/deployments/kubernetes/reloader.yaml

# Mysql
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=keystone | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=nova | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=placement | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=neutron | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=glance | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=skyline | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=mistral | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=octavia | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=barbican | kubectl apply -f -
frep k8s/mysql.yaml.in:- --load config/config.yaml --env db_name=designate | kubectl apply -f -
kubectl wait --for=condition=available --timeout=60s deployment/mysql-keystone
kubectl wait --for=condition=available --timeout=60s deployment/mysql-nova
kubectl wait --for=condition=available --timeout=60s deployment/mysql-placement
kubectl wait --for=condition=available --timeout=60s deployment/mysql-neutron
kubectl wait --for=condition=available --timeout=60s deployment/mysql-glance
kubectl wait --for=condition=available --timeout=60s deployment/mysql-skyline
kubectl wait --for=condition=available --timeout=60s deployment/mysql-mistral
kubectl wait --for=condition=available --timeout=60s deployment/mysql-octavia
kubectl wait --for=condition=available --timeout=60s deployment/mysql-barbican

# Redis (oslo.cache backend)
frep k8s/redis.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Rabbit
frep k8s/rabbit.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Keystone
# wait for redis because we need it for a working keystone
kubectl wait --for=condition=available --timeout=60s deployment/redis
frep k8s/keystone.yaml.in:- --load config/config.yaml | kubectl apply -f -
kubectl wait --for=condition=available --timeout=60s deployment/keystone

# Keystone bootstrap
apt-get install -y python3-venv
python3 -m venv /opt/oscli || echo "Venv creation failed"
/opt/oscli/bin/pip install --upgrade pip
/opt/oscli/bin/pip install python-openstackclient python-mistralclient python-octaviaclient osc-placement osprofiler redis
/opt/oscli/bin/pip install git+https://github.com/arnaudmorin/neutron-wireguard-client

ln -s /opt/oscli/bin/openstack /usr/local/bin/openstack
mkdir -p /etc/bash_completion.d/
openstack complete > /etc/bash_completion.d/openstack

frep files/openrc.in:/root/openrc_admin --load config/config.yaml \
     -e openrc_user=admin -e openrc_project=admin
frep files/openrc.in:/root/openrc_demo --load config/config.yaml \
     -e openrc_user=demo -e openrc_project=demo
cp files/helper /root/helper

# Create keystone resources
cd keystone-bootstrap
frep keystone-settings.yaml.in:keystone-settings.yaml --load ../config/config.yaml
source /opt/oscli/bin/activate
source /root/openrc_admin
./keystone-bootstrap --settings keystone-settings.yaml
cd ../

# Garage (S3)
# Deployed before glance because glance uses it as its image store backend.
# NOTE(arnaud) we drive the garage binary via 'kubectl exec' because the garage
# image is built FROM scratch and has no /bin/sh to run a job script in-container
frep k8s/garage.yaml.in:- --load config/config.yaml | kubectl apply -f -
kubectl wait --for=condition=available --timeout=120s deployment/garage

# Assign the one-time node layout (a fresh garage node serves no S3 without it)
# See: https://garagehq.deuxfleurs.fr/documentation/reference-manual/layout/
if ! kubectl exec deploy/garage -- /garage layout show | grep -q dc1; then
  garage_node=$(kubectl exec deploy/garage -- /garage node id -q | cut -d@ -f1)
  kubectl exec deploy/garage -- /garage layout assign -z dc1 -c 10G $${garage_node}
  kubectl exec deploy/garage -- /garage layout apply --version 1
fi

# Create a default access key and bucket (idempotent)
kubectl exec deploy/garage -- /garage key info my-app >/dev/null 2>&1 || kubectl exec deploy/garage -- /garage key create my-app
kubectl exec deploy/garage -- /garage bucket info my-bucket >/dev/null 2>&1 || kubectl exec deploy/garage -- /garage bucket create my-bucket
kubectl exec deploy/garage -- /garage bucket allow --read --write my-bucket --key my-app

# Glance S3 backend: dedicated key + bucket, credentials injected into config.yaml
# so glance-conf (rendered just below) can reach the S3 store
kubectl exec deploy/garage -- /garage key info glance >/dev/null 2>&1 || kubectl exec deploy/garage -- /garage key create glance
kubectl exec deploy/garage -- /garage bucket info glance >/dev/null 2>&1 || kubectl exec deploy/garage -- /garage bucket create glance
kubectl exec deploy/garage -- /garage bucket allow --read --write glance --key glance
glance_key=$(kubectl exec deploy/garage -- /garage key info glance --show-secret)
glance_ak=$(echo "$${glance_key}" | awk '/Key ID/ {print $NF}')
glance_sk=$(echo "$${glance_key}" | awk '/Secret key/ {print $NF}')
sed -i "s|glance_s3_key|$${glance_ak}|" config/config.yaml
sed -i "s|glance_s3_secret|$${glance_sk}|" config/config.yaml

# Export the garage S3 credentials to /root for convenience (source it to use the aws cli)
garage_key=$(kubectl exec deploy/garage -- /garage key info my-app --show-secret)
cat <<EOF >/root/garagerc
export AWS_ACCESS_KEY_ID=$(echo "$${garage_key}" | awk '/Key ID/ {print $NF}')
export AWS_SECRET_ACCESS_KEY=$(echo "$${garage_key}" | awk '/Secret key/ {print $NF}')
export AWS_DEFAULT_REGION=RegionOne
export AWS_ENDPOINT_URL=http://s3.$${ip}.xip.opensteak.fr
EOF
chmod 600 /root/garagerc

# Glance
frep k8s/glance.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Placement
frep k8s/placement.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Neutron
frep k8s/neutron.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Nova
frep k8s/nova.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Mistral
frep k8s/mistral.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Skyline
frep k8s/skyline.yaml.in:- --load config/config.yaml | kubectl apply -f -

# Barbican
frep k8s/barbican.yaml.in:- --load config/config.yaml | kubectl apply -f -

kubectl wait --for=condition=available --timeout=60s deployment/nova-api
kubectl wait --for=condition=available --timeout=60s deployment/glance
kubectl wait --for=condition=available --timeout=60s deployment/neutron-api

# Pre-populate openstack
source /root/openrc_admin
source /root/helper

create_flavors
create_image_cirros
create_image_debian

source /root/openrc_demo
create_rules
create_key

# Octavia bootstrap
# ansible-playbook ansible/bootstrap-octavia.yaml

frep files/openrc.in:/root/openrc_octavia --load config/config.yaml \
     -e openrc_user=octavia -e openrc_project=service

source /root/openrc_octavia
set -xv
openstack network create \
          --provider-physical-network provider-octavia \
          --provider-network-type flat \
          lb-mgmt-net

openstack subnet create \
          --network lb-mgmt-net \
          --subnet-range 172.21.0.0/16 \
          --no-dhcp \
          --dns-nameserver 1.1.1.1 \
          --allocation-pool start=172.21.0.16,end=172.21.255.254 \
          lb-mgmt-subnet

openstack security group create octavia-amphora --description "Security group for octavia amphpora"
openstack security group rule create octavia-amphora --ingress --protocol tcp --dst-port 9443 --remote-ip 0.0.0.0/0
openstack security group rule create octavia-amphora --ingress --protocol tcp --dst-port 22 --remote-ip 0.0.0.0/0
openstack security group rule create octavia-amphora --ingress --protocol icmp --remote-ip 0.0.0.0/0

openstack keypair create --private-key /root/.ssh/octavia.priv amphora_keypair

set +xv

curl -fL -o /root/amphora.qcow2 https://tarballs.opendev.org/openstack/octavia/test-images/test-only-amphora-x64-haproxy-ubuntu-noble.qcow2

set -xv
openstack image create \
          --file /root/amphora.qcow2 \
          --container-format bare \
          --disk-format qcow2 \
          --tag amphora \
          --public \
          amphora

set +xv

octavia_flavor=$(openstack flavor show medium -f json | jq -r ".id")
octavia_net=$(openstack network show lb-mgmt-net -f json | jq -r ".id")
octavia_secgroup=$(openstack security group show octavia-amphora -f json | jq -r ".id")

sed -ir "s/amp_flavor_id/$${octavia_flavor}/" config/config.yaml
sed -ir "s/amp_boot_network_list/$${octavia_net}/" config/config.yaml
sed -ir "s/amp_secgroup_list/$${octavia_secgroup}/" config/config.yaml

frep k8s/octavia.yaml.in:- --load config/config.yaml | kubectl apply -f -
kubectl wait --for=condition=available --timeout=60s deployment/octavia-api

echo "done"