Skip to content

Commit 6fc9f09

Browse files
oleksandr-ncoleksandr-nc
authored
Merge pull request #783 from nextcloud/feat/K8s-support
Basic K8s support
2 parents 9b7c794 + e71b6a2 commit 6fc9f09

21 files changed

+3064
-54
lines changed

‎.github/workflows/tests-deploy-k8s-clusterip.yml‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
2+
# SPDX-License-Identifier: MIT
3+
#
4+
# K8s ClusterIP expose type tests.
5+
#
6+
# ClusterIP is the default K8s expose type. HaRP creates a ClusterIP Service
7+
# and uses the Service's spec.clusterIP address for upstream routing. This
8+
# works when HaRP runs outside the cluster (e.g. Docker on the host) because
9+
# kube-proxy makes ClusterIPs routable via iptables/ipvs.
10+
#
11+
name: Tests - K8s Deploy (ClusterIP)
12+
13+
on:
14+
pull_request:
15+
branches: [main]
16+
push:
17+
branches: [main]
18+
workflow_dispatch:
19+
20+
permissions:
21+
contents: read
22+
23+
concurrency:
24+
group: tests-deploy-k8s-clusterip-${{ github.head_ref || github.run_id }}
25+
cancel-in-progress: true
26+
27+
env:
28+
HP_SHARED_KEY: 'test_shared_key_12345'
29+
30+
jobs:
31+
k8s-deploy-clusterip:
32+
runs-on: ubuntu-22.04
33+
name: K8s Deploy Lifecycle (ClusterIP)
34+
35+
services:
36+
postgres:
37+
image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest # zizmor: ignore[unpinned-images]
38+
ports:
39+
- 4444:5432/tcp
40+
env:
41+
POSTGRES_USER: root
42+
POSTGRES_PASSWORD: rootpassword
43+
POSTGRES_DB: nextcloud
44+
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
45+
46+
steps:
47+
- name: Set app env
48+
run: echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV
49+
50+
- name: Checkout server
51+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
52+
with:
53+
persist-credentials: false
54+
submodules: true
55+
repository: nextcloud/server
56+
ref: master
57+
58+
- name: Checkout AppAPI
59+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
60+
with:
61+
persist-credentials: false
62+
path: apps/${{ env.APP_NAME }}
63+
64+
- name: Set up php
65+
uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # v2
66+
with:
67+
php-version: '8.3'
68+
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql
69+
coverage: none
70+
ini-file: development
71+
env:
72+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
73+
74+
- name: Check composer file existence
75+
id: check_composer
76+
uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v2
77+
with:
78+
files: apps/${{ env.APP_NAME }}/composer.json
79+
80+
- name: Set up dependencies
81+
if: steps.check_composer.outputs.files_exists == 'true'
82+
working-directory: apps/${{ env.APP_NAME }}
83+
run: composer i
84+
85+
- name: Set up Nextcloud
86+
env:
87+
DB_PORT: 4444
88+
run: |
89+
mkdir data
90+
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 \
91+
--database-port=$DB_PORT --database-user=root --database-pass=rootpassword \
92+
--admin-user admin --admin-pass admin
93+
./occ config:system:set loglevel --value=0 --type=integer
94+
./occ config:system:set debug --value=true --type=boolean
95+
./occ app:enable --force ${{ env.APP_NAME }}
96+
97+
- name: Install k3s
98+
run: |
99+
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb" sh -
100+
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
101+
echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
102+
103+
- name: Wait for k3s and create namespace
104+
run: |
105+
kubectl wait --for=condition=Ready node --all --timeout=120s
106+
kubectl create namespace nextcloud-exapps
107+
NODE_IP=$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
108+
echo "NODE_IP=${NODE_IP}" >> $GITHUB_ENV
109+
echo "k3s node IP: $NODE_IP"
110+
111+
- name: Configure Nextcloud for k3s networking
112+
run: |
113+
./occ config:system:set overwrite.cli.url --value "http://${{ env.NODE_IP }}" --type=string
114+
./occ config:system:set trusted_domains 1 --value "${{ env.NODE_IP }}"
115+
116+
- name: Create K8s service account for HaRP
117+
run: |
118+
kubectl -n nextcloud-exapps create serviceaccount harp-sa
119+
kubectl create clusterrolebinding harp-admin \
120+
--clusterrole=cluster-admin \
121+
--serviceaccount=nextcloud-exapps:harp-sa
122+
K3S_TOKEN=$(kubectl -n nextcloud-exapps create token harp-sa --duration=2h)
123+
echo "K3S_TOKEN=${K3S_TOKEN}" >> $GITHUB_ENV
124+
125+
- name: Pre-pull ExApp image into k3s
126+
run: sudo k3s ctr images pull ghcr.io/nextcloud/app-skeleton-python:latest
127+
128+
- name: Pull HaRP image
129+
run: docker pull ghcr.io/nextcloud/nextcloud-appapi-harp:latest
130+
131+
- name: Start HaRP with K8s backend
132+
run: |
133+
docker run --net host --name appapi-harp \
134+
-e HP_SHARED_KEY="${{ env.HP_SHARED_KEY }}" \
135+
-e NC_INSTANCE_URL="http://${{ env.NODE_IP }}" \
136+
-e HP_LOG_LEVEL="debug" \
137+
-e HP_K8S_ENABLED="true" \
138+
-e HP_K8S_API_SERVER="https://127.0.0.1:6443" \
139+
-e HP_K8S_BEARER_TOKEN="${{ env.K3S_TOKEN }}" \
140+
-e HP_K8S_NAMESPACE="nextcloud-exapps" \
141+
-e HP_K8S_VERIFY_SSL="false" \
142+
--restart unless-stopped \
143+
-d ghcr.io/nextcloud/nextcloud-appapi-harp:latest
144+
145+
- name: Start nginx proxy
146+
run: |
147+
docker run --net host --name nextcloud --rm \
148+
-v $(pwd)/apps/${{ env.APP_NAME }}/tests/simple-nginx-NOT-FOR-PRODUCTION.conf:/etc/nginx/conf.d/default.conf:ro \
149+
-d nginx
150+
151+
- name: Start Nextcloud
152+
run: PHP_CLI_SERVER_WORKERS=2 php -S 0.0.0.0:8080 &
153+
154+
- name: Wait for HaRP K8s readiness
155+
run: |
156+
for i in $(seq 1 30); do
157+
if curl -sf http://${{ env.NODE_IP }}:8780/exapps/app_api/info \
158+
-H "harp-shared-key: ${{ env.HP_SHARED_KEY }}" 2>/dev/null | grep -q '"kubernetes"'; then
159+
echo "HaRP is ready with K8s backend"
160+
exit 0
161+
fi
162+
echo "Waiting for HaRP... ($i/30)"
163+
sleep 2
164+
done
165+
echo "HaRP K8s readiness check failed"
166+
docker logs appapi-harp
167+
exit 1
168+
169+
- name: Register K8s daemon (ClusterIP)
170+
run: |
171+
./occ app_api:daemon:register \
172+
k8s_test "K8s Test" "kubernetes-install" "http" "${{ env.NODE_IP }}:8780" "http://${{ env.NODE_IP }}" \
173+
--harp --harp_shared_key "${{ env.HP_SHARED_KEY }}" --harp_frp_address "${{ env.NODE_IP }}:8782" \
174+
--k8s --k8s_expose_type=clusterip --set-default
175+
./occ app_api:daemon:list
176+
177+
- name: Run K8s integration tests (ClusterIP)
178+
env:
179+
K8S_EXPOSE_TYPE: clusterip
180+
run: python3 apps/${{ env.APP_NAME }}/tests/test_occ_commands_k8s.py
181+
182+
- name: Collect HaRP logs
183+
if: always()
184+
run: docker logs appapi-harp > harp.log 2>&1
185+
186+
- name: Collect K8s resources
187+
if: always()
188+
run: |
189+
kubectl -n nextcloud-exapps get all -o wide > k8s-resources.txt 2>&1 || true
190+
kubectl -n nextcloud-exapps describe pods > k8s-pods-describe.txt 2>&1 || true
191+
kubectl -n nextcloud-exapps get pvc -o wide >> k8s-resources.txt 2>&1 || true
192+
193+
- name: Show all logs
194+
if: always()
195+
run: |
196+
echo "=== HaRP logs ===" && cat harp.log || true
197+
echo "=== K8s resources ===" && cat k8s-resources.txt || true
198+
echo "=== K8s pods ===" && cat k8s-pods-describe.txt || true
199+
echo "=== Nextcloud log (last 100 lines) ===" && tail -100 data/nextcloud.log || true
200+
201+
- name: Upload HaRP logs
202+
if: always()
203+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
204+
with:
205+
name: k8s_deploy_clusterip_harp.log
206+
path: harp.log
207+
if-no-files-found: warn
208+
209+
- name: Upload K8s resources
210+
if: always()
211+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
212+
with:
213+
name: k8s_deploy_clusterip_resources.txt
214+
path: |
215+
k8s-resources.txt
216+
k8s-pods-describe.txt
217+
if-no-files-found: warn
218+
219+
- name: Upload NC logs
220+
if: always()
221+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
222+
with:
223+
name: k8s_deploy_clusterip_nextcloud.log
224+
path: data/nextcloud.log
225+
if-no-files-found: warn
226+
227+
tests-success:
228+
permissions:
229+
contents: none
230+
runs-on: ubuntu-22.04
231+
needs: [k8s-deploy-clusterip]
232+
name: K8s-ClusterIP-Tests-OK
233+
steps:
234+
- run: echo "K8s ClusterIP tests passed successfully"

0 commit comments

Comments
 (0)