Skip to content

Commit 1b27500

Browse files
SimonLoirSimonLoir
authored
feat: improve backup process (#55)
* feat: enhance volume backup processes with job scheduler and backup run tracking - Integrated queue job scheduler for creating and deleting volume backup schedules. - Added backup run tracking with status updates (`RUNNING`, `SUCCESS`, `FAILED`) in database. - Improved API procedures for volume backup schedule creation and deletion. * feat: add centralized logging utility and enhance volume backup logging - Introduced `getLogger` utility for capturing and managing logs with structured formatting. - Integrated `getLogger` into volume backup workers for consistent logging across processes. - Added `logs` field to `BackupRun` in Prisma schema to store encrypted logs for better debugging and tracking. - Updated volume backup flow to capture structured logs, improve error handling, and store logs in the database. - Created migration to add `logs` column to `BackupRun` table. * feat: enhance volume backup with S3 utilities and improved retention handling - Added `generateRcloneFlags` utility for building dynamic rclone command flags. - Introduced `getS3Storage` utility for fetching S3 destinations from the database. - Enhanced volume backup flow to upload backups to dynamically organized S3 paths using `rclone copyto`. - Integrated retention handling with copies created for each rule defined in schedules. - Updated local backup cleanup process to improve storage hygiene. * feat: add Docker integration for workers and Redis setup - Introduced a multi-stage Dockerfile for the workers app with dependency installation, build, and runtime layers. - Added GitHub Actions workflow to build and push worker Docker images, supporting multiple architectures (amd64, arm64). - Updated `docker-compose.test.yml` to include worker and Redis services for local testing. - Expanded `apps/workers/package.json` dependencies with logging (`pino`, `pino-pretty`), Prisma client, and PostgreSQL (`pg`). - Adjusted `pnpm-lock.yaml` to reflect new dependencies. * refactor: simplify `volumeBackupsQueue` imports in backup procedures - Updated imports in `createVolumeBackup` and `deleteVolumeBackupSchedule` to use `@repo/queues` directly instead of `@repo/queues/dist`. * chore(web): bump version to 0.11.4 in package.json
1 parent a3a07c2 commit 1b27500

17 files changed

Lines changed: 449 additions & 52 deletions

File tree

‎.github/workflows/build-and-push-worker-image.yml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
name: Build and Push Docker Image
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'apps/web/package.json'
9+
10+
env:
11+
REGISTRY: ghcr.io
12+
IMAGE_NAME: seastackapp/seastack-worker
13+
14+
jobs:
15+
check-version-change:
16+
runs-on: ubuntu-latest
17+
permissions:
18+
contents: read
19+
outputs:
20+
version_changed: ${{ steps.check.outputs.changed }}
21+
new_version: ${{ steps.get_version.outputs.version }}
22+
steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4.2.2
25+
with:
26+
fetch-depth: 2
27+
28+
- name: Get current version
29+
id: get_version
30+
run: |
31+
VERSION=$(jq -r '.version' apps/web/package.json)
32+
echo "version=$VERSION" >> $GITHUB_OUTPUT
33+
echo "Current version: $VERSION"
34+
35+
- name: Get previous version
36+
id: get_prev_version
37+
run: |
38+
if git show HEAD^1:apps/web/package.json > /dev/null 2>&1; then
39+
PREV_VERSION=$(git show HEAD^1:apps/web/package.json | jq -r '.version')
40+
else
41+
PREV_VERSION="none"
42+
fi
43+
echo "prev_version=$PREV_VERSION" >> $GITHUB_OUTPUT
44+
echo "Previous version: $PREV_VERSION"
45+
46+
- name: Check if version changed
47+
id: check
48+
run: |
49+
CURRENT="${{ steps.get_version.outputs.version }}"
50+
PREVIOUS="${{ steps.get_prev_version.outputs.prev_version }}"
51+
if [ "$CURRENT" != "$PREVIOUS" ]; then
52+
echo "changed=true" >> $GITHUB_OUTPUT
53+
echo "Version changed from $PREVIOUS to $CURRENT"
54+
else
55+
echo "changed=false" >> $GITHUB_OUTPUT
56+
echo "Version did not change"
57+
fi
58+
59+
build-and-push:
60+
needs: check-version-change
61+
if: needs.check-version-change.outputs.version_changed == 'true'
62+
strategy:
63+
matrix:
64+
include:
65+
- runner: ubuntu-latest
66+
platform: linux/amd64
67+
- runner: ubuntu-24.04-arm
68+
platform: linux/arm64
69+
runs-on: ${{ matrix.runner }}
70+
permissions:
71+
contents: read
72+
packages: write
73+
steps:
74+
- name: Checkout code
75+
uses: actions/checkout@v4.2.2
76+
77+
- name: Set up Docker Buildx
78+
uses: docker/setup-buildx-action@v3.7.1
79+
80+
- name: Log in to GitHub Container Registry
81+
uses: docker/login-action@v3.3.0
82+
with:
83+
registry: ghcr.io
84+
username: ${{ github.actor }}
85+
password: ${{ secrets.GITHUB_TOKEN }}
86+
87+
- name: Build and push by digest
88+
id: build
89+
uses: docker/build-push-action@v6.10.0
90+
with:
91+
context: .
92+
file: ./apps/workers/Dockerfile
93+
platforms: ${{ matrix.platform }}
94+
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
95+
cache-from: type=gha
96+
cache-to: type=gha,mode=max
97+
98+
- name: Export digest
99+
run: |
100+
mkdir -p /tmp/digests
101+
digest="${{ steps.build.outputs.digest }}"
102+
touch "/tmp/digests/${digest#sha256:}"
103+
104+
- name: Upload digest
105+
uses: actions/upload-artifact@v4
106+
with:
107+
name: digests-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
108+
path: /tmp/digests/*
109+
if-no-files-found: error
110+
retention-days: 1
111+
112+
merge:
113+
needs: [check-version-change, build-and-push]
114+
runs-on: ubuntu-latest
115+
permissions:
116+
contents: read
117+
packages: write
118+
steps:
119+
- name: Download digests
120+
uses: actions/download-artifact@v4
121+
with:
122+
path: /tmp/digests
123+
pattern: digests-*
124+
merge-multiple: true
125+
126+
- name: Set up Docker Buildx
127+
uses: docker/setup-buildx-action@v3.7.1
128+
129+
- name: Log in to GitHub Container Registry
130+
uses: docker/login-action@v3.3.0
131+
with:
132+
registry: ghcr.io
133+
username: ${{ github.actor }}
134+
password: ${{ secrets.GITHUB_TOKEN }}
135+
136+
- name: Extract metadata
137+
id: meta
138+
uses: docker/metadata-action@v5.6.1
139+
with:
140+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
141+
tags: |
142+
type=raw,value=latest
143+
type=raw,value=${{ needs.check-version-change.outputs.new_version }}
144+
145+
- name: Create manifest list and push
146+
working-directory: /tmp/digests
147+
run: |
148+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
149+
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)

‎apps/web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "web",
3-
"version": "0.11.3",
3+
"version": "0.11.4",
44
"type": "module",
55
"private": true,
66
"scripts": {

‎apps/workers/Dockerfile‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Multi-stage Dockerfile for the workers app in a pnpm + turbo monorepo
2+
# Uses Alpine for the final worker image
3+
4+
# -----------------------
5+
# 1) Base image with pnpm
6+
# -----------------------
7+
FROM node:24-alpine AS base
8+
9+
# Runtime dependencies commonly needed by Node native modules & Prisma
10+
RUN apk add --no-cache libc6-compat openssl
11+
12+
# Enable corepack and pin pnpm (must match repo tooling)
13+
RUN corepack enable && corepack prepare pnpm@9.0.0 --activate
14+
15+
WORKDIR /app
16+
17+
# -----------------------
18+
# 2) Dependencies layer
19+
# - install workspace deps with caching
20+
# -----------------------
21+
FROM base AS deps
22+
23+
# Build toolchain for native modules (not in final image)
24+
RUN apk add --no-cache python3 make g++
25+
26+
# Copy only files needed to resolve the dependency graph to leverage Docker cache
27+
COPY pnpm-lock.yaml pnpm-workspace.yaml package.json turbo.json ./
28+
29+
# Copy workspace manifests used by the workers app and its internal deps
30+
COPY apps/workers/package.json ./apps/workers/package.json
31+
COPY packages ./packages
32+
33+
# Install dependencies (workspace-aware) using lockfile
34+
RUN pnpm install --frozen-lockfile
35+
36+
# -----------------------
37+
# 3) Build layer
38+
# -----------------------
39+
FROM deps AS build
40+
41+
# Bring in full source to build the workers package
42+
COPY . .
43+
44+
# Provide safe defaults for codegen steps that don't require a live DB
45+
ENV DATABASE_URL=postgres://postgres:password@postgres:5432/postgres
46+
47+
# Generate Prisma client for @repo/db (no live DB needed)
48+
RUN pnpm --filter @repo/db... generate || pnpm --filter @repo/db generate || true
49+
50+
# Build the workers app (outputs to apps/workers/dist)
51+
RUN pnpm --filter workers build
52+
53+
# Produce a pruned, production-ready output for just the workers package
54+
# This includes the package files, its dist, and a pruned node_modules
55+
RUN pnpm deploy --filter workers --prod /out
56+
57+
# -----------------------
58+
# 4) Runtime layer (Alpine)
59+
# -----------------------
60+
FROM node:24-alpine AS runner
61+
62+
ENV NODE_ENV=production
63+
64+
# Runtime libs needed by Prisma and other native deps
65+
RUN apk add --no-cache libc6-compat openssl
66+
67+
# Non-root user for security
68+
RUN adduser -D worker
69+
70+
WORKDIR /app
71+
72+
# Copy the pruned deployment from build stage
73+
COPY --from=build /out/ .
74+
75+
USER worker
76+
77+
# Start the worker process
78+
CMD ["node", "dist/index.mjs"]

‎apps/workers/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,16 @@
1414
"dependencies": {
1515
"@dotenvx/dotenvx": "^1.51.1",
1616
"@prisma/adapter-pg": "^7.1.0",
17+
"@prisma/client": "^7.1.0",
1718
"@repo/db": "workspace:*",
1819
"@repo/queues": "workspace:*",
1920
"@repo/utils": "workspace:*",
2021
"@types/ssh2": "^1.15.5",
2122
"bullmq": "^5.65.1",
2223
"ioredis": "^5.8.2",
24+
"pino": "^10.1.0",
25+
"pino-pretty": "^13.1.3",
26+
"pg": "^8.16.3",
2327
"prisma": "^7.1.0",
2428
"ssh2": "^1.17.0"
2529
},

‎apps/workers/src/backups.ts‎

Lines changed: 61 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,22 @@ import { setupWorker } from './setupWorker';
22
import { BACKUPS_QUEUE_NAME, VolumeBackupJob } from '@repo/queues';
33
import { prisma } from '@repo/db';
44
import {
5-
decrypt,
5+
encrypt,
6+
generateRcloneFlags,
67
generateVolumeName,
8+
getLogger,
9+
getS3Storage,
710
getSSHClient,
11+
parseRetentionString,
812
remoteExec,
913
sh,
1014
} from '@repo/utils';
1115
import { Client } from 'ssh2';
1216

1317
export const setUpVolumeBackups = () => {
1418
return setupWorker<VolumeBackupJob>(BACKUPS_QUEUE_NAME, async (job) => {
15-
console.log(`Processing job ${job.id}`);
19+
const { logger, logs } = getLogger();
20+
console.info(`Processing job ${job.id}`);
1621
const schedule = await prisma.volumeBackupSchedule.findUnique({
1722
where: { id: job.data.schedule },
1823
include: {
@@ -34,13 +39,22 @@ export const setUpVolumeBackups = () => {
3439
console.error(`Could not find schedule ${job.data.schedule}`);
3540
return;
3641
}
37-
console.log(
42+
console.info(
3843
`Starting backup for schedule ${schedule.id} (${schedule.volume.name})`
3944
);
4045
const service = schedule.volume.service;
4146
const serverId = service.server.id;
4247
const volumeName = generateVolumeName(schedule.volume.name, service.id);
43-
const backupFilename = `backup-${volumeName}-${job.id}.tar.zst`;
48+
const baseFileName = `backup-${volumeName}`;
49+
const backupFilename = `${baseFileName}.tar.zst`;
50+
const run = await prisma.backupRun.create({
51+
data: {
52+
status: 'RUNNING',
53+
volumeBackupSchedule: {
54+
connect: { id: schedule.id },
55+
},
56+
},
57+
});
4458

4559
let connection: Client | undefined = undefined;
4660
try {
@@ -49,46 +63,59 @@ export const setUpVolumeBackups = () => {
4963
serverId,
5064
schedule.volume.service.server.organizations[0]!.id
5165
);
52-
console.log('Connected to server via SSH');
66+
logger.debug('Connected to server via SSH');
5367

54-
const command = sh`docker run --rm -v ${volumeName}:/data alpine sh -c "tar -C /data -cf - ." | zstd -z -19 -o ${backupFilename}`;
68+
const command = sh`docker run --rm -v ${volumeName}:/data alpine sh -c "tar -C /data -cf - ." | zstd -z -19 -o ${backupFilename} -f`;
5569

56-
console.log(`Running command: ${command}`);
70+
logger.debug(`Running command: ${command}`);
5771
await remoteExec(connection, command);
58-
console.log(`Backup created: ${backupFilename}`);
5972

60-
console.log('Uploading backup file to S3 using rclone');
73+
logger.debug('Uploading backup file to S3 using rclone');
6174

62-
const s3 = await prisma.s3Storage.findFirst({
63-
where: { id: schedule.storageDestinationId },
64-
});
75+
const s3 = await getS3Storage(
76+
prisma,
77+
schedule.storageDestinationId
78+
);
6579

66-
if (!s3) {
67-
throw new Error(
68-
`Could not find S3 storage destination ${schedule.storageDestinationId}`
69-
);
70-
}
80+
const flags = generateRcloneFlags(s3);
81+
const target = sh`:s3:${s3.bucket}/seastack/backups/${schedule.id}/${new Date().getTime()}-${backupFilename}`;
82+
const secureName = sh`./${backupFilename}`;
83+
const rcloneCommand = `rclone copyto ${secureName} ${target} ${flags} --progress`;
7184

72-
const flags = [
73-
'--s3-provider=Other',
74-
sh`--s3-access-key-id=${decrypt(s3.accessKeyId)}`,
75-
sh`--s3-secret-access-key=${decrypt(s3.secretAccessKey)}`,
76-
sh`--s3-endpoint=${s3.endpoint}`,
77-
s3.region ? `--s3-region=${s3.region}` : '',
78-
'--s3-acl=private',
79-
sh`--s3-force-path-style=${s3.usePathStyle ? 'true' : 'false'}`,
80-
].join(' ');
81-
const target = sh`:s3:${s3.bucket}`;
82-
const secureName = sh`${backupFilename}`;
83-
const rcloneCommand = `rclone copy ${secureName} ${target} ${flags} --progress`;
85+
logger.info(`Running command: ${rcloneCommand}`);
86+
logger.info(await remoteExec(connection, rcloneCommand));
8487

85-
console.log(`Running command: ${rcloneCommand}`);
86-
console.log(await remoteExec(connection, rcloneCommand));
88+
logger.info('Creating copies for data retention');
89+
const { rules } = parseRetentionString(schedule.retention);
90+
for (const { unit } of rules) {
91+
if (unit === 'latest') continue;
92+
const newTarget = sh`:s3:${s3.bucket}/seastack/backups/${schedule.id}/${baseFileName}.${unit}`;
8793

88-
console.log('Deleting local backup file');
89-
await remoteExec(connection, sh`rm ${backupFilename}`);
94+
const copyCommand = `rclone copyto ${target} ${newTarget} ${flags} --progress`;
95+
logger.info(`Running command: ${copyCommand}`);
96+
logger.info(await remoteExec(connection, copyCommand));
97+
}
98+
99+
logger.debug('Deleting local backup file');
100+
logger.debug(
101+
await remoteExec(connection, sh`rm ${backupFilename}`)
102+
);
103+
104+
logger.info(`Backup created: ${backupFilename}`);
105+
await prisma.backupRun.update({
106+
where: { id: run.id },
107+
data: {
108+
status: 'SUCCESS',
109+
artifactLocation: backupFilename,
110+
logs: encrypt(logs.join('')),
111+
},
112+
});
90113
} catch (error) {
91-
console.error(error);
114+
logger.error(error);
115+
await prisma.backupRun.update({
116+
where: { id: run.id },
117+
data: { status: 'FAILED', logs: encrypt(logs.join('')) },
118+
});
92119
throw error;
93120
} finally {
94121
if (connection) connection.end();

0 commit comments

Comments
 (0)