Skip to content

Commit a047f2b

Browse files
nmburgannmburgan
committed
Refactor Vox rake tasks
This moves our tasks into the rakelib dir, which is where rake tasks are supposed to live and is automatically loaded, so we don't need a loading line in the Rakefile. This refactors the tasks to extract common methods around running shell commands and running things inside a container. It also significantly reorganizes the build task to make it a bit clearer what each step of the build process is doing.
1 parent 63efac5 commit a047f2b

File tree

10 files changed

+361
-297
lines changed

10 files changed

+361
-297
lines changed

‎Rakefile‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,4 @@
11
require 'rake'
2-
require 'open3'
3-
4-
RED = "\033[31m"
5-
GREEN = "\033[32m"
6-
RESET = "\033[0m"
7-
8-
def run_command(cmd, silent: true, print_command: false, report_status: false, allowed_exit_codes: [0])
9-
puts "#{GREEN}Running #{cmd}#{RESET}" if print_command
10-
output = ''
11-
Open3.popen2e(cmd) do |_stdin, stdout_stderr, thread|
12-
stdout_stderr.each do |line|
13-
puts line unless silent
14-
output += line
15-
end
16-
exitcode = thread.value.exitstatus
17-
unless allowed_exit_codes.include?(exitcode)
18-
err = "#{RED}Command failed! Command: #{cmd}, Exit code: #{exitcode}"
19-
# Print details if we were running silent
20-
err += "\nOutput:\n#{output}" if silent
21-
err += RESET
22-
abort err
23-
end
24-
puts "#{GREEN}Command finished with status #{exitcode}#{RESET}" if report_status
25-
end
26-
output.chomp
27-
end
28-
29-
Dir.glob(File.join('tasks/**/*.rake')).each { |file| load file }
302

313
### puppetlabs stuff ###
324
def run_beaker(test_files)
@@ -211,30 +183,3 @@ namespace :release do
211183
end
212184
end
213185
end
214-
215-
begin
216-
require 'github_changelog_generator/task'
217-
rescue LoadError
218-
task :changelog do
219-
abort('Run `bundle install --with release` to install the `github_changelog_generator` gem.')
220-
end
221-
else
222-
GitHubChangelogGenerator::RakeTask.new :changelog do |config|
223-
config.header = <<~HEADER.chomp
224-
# Changelog
225-
226-
All notable changes to this project will be documented in this file.
227-
HEADER
228-
config.user = 'openvoxproject'
229-
config.project = 'openvoxdb'
230-
config.exclude_labels = %w[dependencies duplicate question invalid wontfix wont-fix modulesync skip-changelog]
231-
# this is probably the worst way to do this
232-
# ideally there would be a VERSION file and clojure and Rake would read it
233-
config.future_release = File.readlines('project.clj').first.scan(/".*"/).first.gsub('"', '')
234-
# we limit the changelog to all new openvox releases, to skip perforce onces
235-
# otherwise the changelog generate takes a lot amount of time
236-
config.since_tag = '8.9.1'
237-
#config.exclude_tags_regex = /\A7\./
238-
config.release_branch = 'main'
239-
end
240-
end

‎rakelib/build.rake‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
# frozen_string_literal: true
2+
3+
# Builds packages using a Docker container
4+
# Env var options:
5+
# NO_TEARDOWN - If set, do not stop the docker container after the build for debugging purposes.
6+
# EZBAKE_BRANCH - If set, use this ezbake branch for building. If used, must also set EZBAKE_VERSION accordingly.
7+
# EZBAKE_REPO - If EZBAKE_BRANCH is set, use this repo URL for ezbake (default: https://github.com/openvoxproject/ezbake)
8+
# EZBAKE_VERSION - If set, use this version string in project.clj for ezbake. Must correspond to the version currently in defproject
9+
# in EZBAKE_BRANCH.
10+
# FULL_DEP_REBUILD_BRANCH - If set, rebuild all dependencies, where all dependency repos have this branch present with your
11+
# desired changes. Overrides any other DEP_ settings.
12+
# DEP_REBUILD - Comma-separated list of dependencies to rebuild from source. Should be the repo names.
13+
# DEP_REBUILD_BRANCH - If DEP_REBUILD is set, use this branch for the specified dependencies (default: main)
14+
# DEP_REBUILD_ORG - If DEP_REBUILD is set, use this GitHub org for the specified dependencies to build the repo URL (default: openvoxproject)
15+
# DEB_PLATFORMS - Comma-separated list of debian/ubuntu platforms to build for
16+
# RPM_PLATFORMS - Comma-separated list of rpm platforms to build for
17+
# FIPS - If set, build specified platforms with the appropriate 'fips' lein profile(s) enabled.
18+
19+
require 'fileutils'
20+
require 'tmpdir'
21+
require_relative 'utils/docker_runner'
22+
23+
module Vox
24+
class Build
25+
def platform_targets
26+
# It seems like these are special files/names that, when you want to add a new one, require
27+
# changes in some other component. But no, it seems to only really look at the parts of
28+
# the text in the string, as long as it looks like "base-<whatever you want to call the platform>-i386.cow"
29+
# and "<doesn't matter>-<os>-<osver>-<arch which doesn't matter because it's actually noarch>".
30+
# I think it just treats all debs like Debian these days. And all rpms are similar.
31+
# So do whatever you want I guess. We really don't need separate packages for each platform.
32+
# To be fixed one of these days. Relevant stuff:
33+
# https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/tasks/build.rake
34+
# https://github.com/puppetlabs/ezbake/blob/aeb7735a16d2eecd389a6bd9e5c0cfc7c62e61a5/resources/puppetlabs/lein-ezbake/template/global/ext/fpm.rb
35+
deb_platforms = ENV['DEB_PLATFORMS'] || 'ubuntu-20.04,ubuntu-22.04,ubuntu-24.04,ubuntu-25.04,debian-11,debian-12,debian-13'
36+
rpm_platforms = ENV['RPM_PLATFORMS'] || 'el-8,el-9,el-10,sles-15,sles-16,amazon-2,amazon-2023,fedora-42,fedora-43'
37+
38+
debs = deb_platforms.split(',').map { |p| "base-#{p.split("-").join}-i386.cow" }.join(" ")
39+
rpms = rpm_platforms.split(',').map { |p| "pl-#{p}-x86_64" }.join(" ")
40+
41+
[debs, rpms]
42+
end
43+
# The deps must be built in this order due to dependencies between them.
44+
# There is a circular dependency between clj-http-client and trapperkeeper-webserver-jetty10,
45+
# but only for tests, so the build *should* work.
46+
DEP_BUILD_ORDER = [
47+
'clj-parent',
48+
'clj-kitchensink',
49+
'clj-i18n',
50+
'comidi',
51+
'jvm-ssl-utils',
52+
'trapperkeeper',
53+
'trapperkeeper-filesystem-watcher',
54+
'trapperkeeper-webserver-jetty10',
55+
'trapperkeeper-authorization',
56+
'trapperkeeper-metrics',
57+
'trapperkeeper-status',
58+
'stockpile',
59+
'structured-logging',
60+
].freeze
61+
62+
def initialize(tag:)
63+
@tag = tag
64+
@runner = Vox::DockerRunner.new(container_name: 'openvoxdb-builder', image: 'ezbake-builder')
65+
@deps_tmp = Dir.mktmpdir('deps')
66+
end
67+
68+
def build
69+
checkout_tag_if_requested
70+
build_image_unless_present
71+
start_container
72+
build_and_install_libs(prepare_deps)
73+
build_project
74+
postprocess_output
75+
ensure
76+
@runner.teardown unless ENV['NO_TEARDOWN']
77+
FileUtils.rm_rf(@deps_tmp)
78+
end
79+
80+
private
81+
82+
def checkout_tag_if_requested
83+
if @tag.nil? || @tag.empty?
84+
puts 'Running build with current branch'
85+
else
86+
puts "Running build on #{@tag}"
87+
Vox::Shell.run("git fetch --tags && git checkout #{@tag}")
88+
end
89+
end
90+
91+
def build_image_unless_present
92+
return if @runner.image_exists?
93+
94+
# If the Dockerfile has changed since this was last built,
95+
# delete all containers and do `docker rmi ezbake-builder`
96+
puts 'Building ezbake-builder image'
97+
Vox::Shell.run("docker build -t ezbake-builder .", silent: false, print_command: true)
98+
end
99+
100+
def prepare_deps
101+
libs = {}
102+
103+
# Manage ezbake override
104+
ezbake_branch = ENV.fetch('EZBAKE_BRANCH', '').strip
105+
unless ezbake_branch.empty?
106+
libs['ezbake'] = {
107+
repo: ENV.fetch('EZBAKE_REPO', 'https://github.com/openvoxproject/ezbake'),
108+
branch: ENV.fetch('EZBAKE_BRANCH', 'main'),
109+
}
110+
end
111+
112+
# Decide if we're rebuilding everything or a subset
113+
full_rebuild_branch = ENV.fetch('FULL_DEP_REBUILD_BRANCH', '').strip
114+
subset_list = ENV.fetch('DEP_REBUILD', '').split(',').map(&:strip).reject(&:empty?)
115+
subset_branch = ENV.fetch('DEP_REBUILD_BRANCH', 'main').strip
116+
rebuild_org = ENV.fetch('DEP_REBUILD_ORG', 'openvoxproject').strip
117+
118+
selected_libs = []
119+
selected_branch = nil
120+
121+
if !full_rebuild_branch.empty?
122+
selected_branch = full_rebuild_branch
123+
selected_libs = DEP_BUILD_ORDER.dup
124+
elsif !subset_list.empty?
125+
selected_branch = subset_branch
126+
unknown = subset_list - DEP_BUILD_ORDER
127+
puts "WARNING: Unknown deps in DEP_REBUILD (will be ignored): #{unknown.join(', ')}" unless unknown.empty?
128+
selected_libs = DEP_BUILD_ORDER & subset_list # Keeps DEP_BUILD_ORDER ordering
129+
end
130+
131+
selected_libs.each do |lib|
132+
libs[lib] = { repo: "https://github.com/#{rebuild_org}/#{lib}", branch: selected_branch }
133+
end
134+
135+
libs.each do |lib, config|
136+
puts "Checking out #{lib}"
137+
Vox::Shell.run(
138+
"git clone --revision #{config[:branch]} #{config[:repo]} #{@deps_tmp}/#{lib}",
139+
silent: false,
140+
print_command: true
141+
)
142+
end
143+
144+
libs
145+
end
146+
147+
def start_container
148+
@runner.teardown if @runner.container_exists?
149+
volumes = [[Dir.pwd, '/code'], [@deps_tmp, '/deps']]
150+
@runner.start(volumes: volumes)
151+
end
152+
153+
def build_and_install_libs(libs)
154+
libs.each_key do |lib|
155+
puts "Building and installing #{lib} from source"
156+
@runner.exec("cd /deps/#{lib} && lein install")
157+
end
158+
end
159+
160+
def build_project
161+
debs, rpms = platform_targets
162+
fips = !ENV['FIPS'].nil?
163+
ezbake_version_var = ENV['EZBAKE_VERSION'] ? "EZBAKE_VERSION=#{ENV['EZBAKE_VERSION']}" : ""
164+
165+
puts 'Building openvoxdb'
166+
@runner.exec('cd /code && rm -rf ruby output && bundle install --without test && lein install')
167+
@runner.exec(
168+
"cd /code && COW=\"#{debs}\" MOCK=\"#{rpms}\" GEM_SOURCE='https://rubygems.org' #{ezbake_version_var} " \
169+
"EZBAKE_ALLOW_UNREPRODUCIBLE_BUILDS=true EZBAKE_NODEPLOY=true LEIN_PROFILES=ezbake " \
170+
"lein with-profile #{fips ? "fips," : ""}user,ezbake,provided,internal ezbake local-build"
171+
)
172+
end
173+
174+
def postprocess_output
175+
Vox::Shell.run("sudo chown -R $USER output", print_command: true)
176+
Dir.glob("output/**/*i386*").each { |f| FileUtils.rm_rf(f) }
177+
Dir.glob("output/puppetdb-*.tar.gz").each { |f| FileUtils.mv(f, f.sub('puppetdb', 'openvoxdb')) }
178+
end
179+
end
180+
end
181+
182+
namespace :vox do
183+
desc 'Build openvoxdb packages with Docker'
184+
task :build, [:tag] do |_, args|
185+
Vox::Build.new(tag: args[:tag]).build
186+
end
187+
end

‎rakelib/changelog.rake‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# frozen_string_literal: true
2+
3+
begin
4+
require 'github_changelog_generator/task'
5+
rescue LoadError
6+
task :changelog do
7+
abort('Run `bundle install --with release` to install the `github_changelog_generator` gem.')
8+
end
9+
else
10+
GitHubChangelogGenerator::RakeTask.new :changelog do |config|
11+
config.header = <<~HEADER.chomp
12+
# Changelog
13+
14+
All notable changes to this project will be documented in this file.
15+
HEADER
16+
config.user = 'openvoxproject'
17+
config.project = 'openvoxdb'
18+
config.exclude_labels = %w[dependencies duplicate question invalid wontfix wont-fix modulesync skip-changelog]
19+
# this is probably the worst way to do this
20+
# ideally there would be a VERSION file and clojure and Rake would read it
21+
config.future_release = File.readlines('project.clj').first.scan(/".*"/).first.gsub('"', '')
22+
# we limit the changelog to all new openvox releases, to skip perforce onces
23+
# otherwise the changelog generate takes a lot amount of time
24+
config.since_tag = '8.9.1'
25+
#config.exclude_tags_regex = /\A7\./
26+
config.release_branch = 'main'
27+
end
28+
end

‎rakelib/test.rake‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# frozen_string_literal: true
2+
3+
# Tests OpenVoxDB in the same manner as we do in PR tests.
4+
# Only tested with a handful of the cells we use in CI. May need adjustment for others.
5+
#
6+
# Env var options:
7+
# NO_TEARDOWN - If set, do not stop the docker container after the tests for debugging purposes.
8+
9+
require_relative 'utils/docker_runner'
10+
11+
module Vox
12+
class Test
13+
def initialize(spec:)
14+
@spec = spec
15+
@runner = Vox::DockerRunner.new(container_name: 'openvoxdb-test', image: select_image(@spec))
16+
end
17+
18+
def run_tests
19+
@runner.teardown if @runner.container_exists?
20+
@runner.start(volumes: [[Dir.pwd, '/code']])
21+
22+
# The tests make tons of temp dirs in the current working directory, and trying to do this
23+
# inside the volume mount dir results in permissions issues. So we copy all of the code to /tmp/code
24+
# and run from there. Not ideal, but we have to since it curls down pdbbox and we don't control that.
25+
@runner.exec('cp -r /code /tmp && chown -R root:root /tmp/code')
26+
@runner.exec('apt update && apt install -y leiningen curl python3 procps')
27+
@runner.exec("cd /tmp/code && rm -rf ci/local && ext/bin/prep-debianish-root --for #{@spec} --install ci/local")
28+
@runner.exec('echo "postgres ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers')
29+
# There is a non-fatal error when running on an arm64 host, so we can ignore exit 255.
30+
@runner.exec("cd /tmp/code && ci/bin/prep-and-run-in local #{@spec}", allowed_exit_codes: [0, 255])
31+
@runner.exec('chown -R postgres:postgres /tmp/code')
32+
@runner.exec("cd /tmp/code && NO_ACCEPTANCE=true ci/bin/run #{@spec}", user: 'postgres')
33+
ensure
34+
@runner.teardown unless ENV['NO_TEARDOWN']
35+
end
36+
37+
private
38+
39+
def select_image(spec)
40+
_suite, java, _pg = spec.split('/')
41+
java =~ /17/ ? 'ruby:3.2-bookworm' : 'ruby:3.2-trixie'
42+
end
43+
end
44+
end
45+
46+
namespace :vox do
47+
desc 'Run lein test locally in the same way that PR checks run with a properly configured postgres and other artifacts.'
48+
task :test, [:spec] do |_, args|
49+
Vox::Test.new(spec: (args[:spec] || 'core+ext/openjdk17/pg-17')).run_tests
50+
end
51+
end

tasks/upload.rake renamed to rakelib/upload.rake

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1+
# frozen_string_literal: true
2+
3+
require_relative 'utils/shell'
4+
15
namespace :vox do
26
desc 'Upload artifacts from the output directory to S3. Requires the AWS CLI to be installed and configured appropriately.'
37
task :upload, [:platform] do |_, args|
48
endpoint = ENV.fetch('ENDPOINT_URL')
59
bucket = ENV.fetch('BUCKET_NAME')
610
component = 'openvoxdb'
711
os = nil
8-
arch = nil
912
if args[:platform]
1013
parts = args[:platform].split('-')
1114
os = parts[0].gsub('fedora','fc') + parts[1]
12-
arch = parts[2]
1315
end
1416

1517
abort 'You must set the ENDPOINT_URL environment variable to the S3 server you want to upload to.' if endpoint.nil? || endpoint.empty?
@@ -18,7 +20,7 @@ namespace :vox do
1820
s3 = "aws s3 --endpoint-url=#{endpoint}"
1921

2022
# Ensure the AWS CLI isn't going to fail with the given parameters
21-
run_command("#{s3} ls s3://#{bucket}/")
23+
Vox::Shell.run("#{s3} ls s3://#{bucket}/")
2224

2325
config = File.expand_path("../target/staging/ezbake.rb", __dir__)
2426
abort "Could not find ezbake config from the build at #{config}" unless File.exist?(config)
@@ -40,7 +42,7 @@ namespace :vox do
4042

4143
path = "s3://#{bucket}/#{component}/#{tag}"
4244
files.each do |f|
43-
run_command("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false)
45+
Vox::Shell.run("#{s3} cp #{f} #{path}/#{File.basename(f)}", silent: false)
4446
end
4547
end
4648
end

0 commit comments

Comments
 (0)