Skip to content

Commit 5b5f2c0

Browse files
henricasanovahenricasanova
authored
Merge pull request #96 from wfcommons/testing
- Added a Pegasus translator test - Fixed Pegasus translator bug
2 parents cab9a8d + e878711 commit 5b5f2c0

File tree

3 files changed

+176
-6
lines changed

3 files changed

+176
-6
lines changed

‎tests/translators/Dockerfile.pegasus‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
# docker build --platform amd64 -t wfcommons-dev-dask -f Dockerfile.dask .
2+
# docker run -it --rm -v `pwd`:/home/wfcommons wfcommons-dev-dask /bin/bash
3+
4+
FROM amd64/ubuntu:noble
5+
6+
LABEL org.containers.image.authors="henric@hawaii.edu"
7+
8+
# update repositories
9+
RUN apt-get update
10+
11+
# set timezone
12+
RUN echo "America/Los_Angeles" > /etc/timezone && export DEBIAN_FRONTEND=noninteractive && apt-get install -y tzdata
13+
14+
# install useful stuff
15+
RUN apt-get -y install pkg-config
16+
RUN apt-get -y install git
17+
RUN apt-get -y install wget
18+
RUN apt-get -y install make
19+
RUN apt-get -y install cmake
20+
RUN apt-get -y install cmake-data
21+
RUN apt-get -y install sudo
22+
RUN apt-get -y install vim --fix-missing
23+
RUN apt-get -y install gcc
24+
RUN apt-get -y install gcc-multilib
25+
26+
# Python stuff
27+
RUN apt-get -y install python3 python3-pip
28+
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
29+
RUN python3 -m pip install --break-system-packages pathos pandas filelock
30+
RUN python3 -m pip install --break-system-packages networkx scipy matplotlib
31+
RUN python3 -m pip install --break-system-packages pyyaml jsonschema requests
32+
RUN python3 -m pip install --break-system-packages --upgrade setuptools
33+
34+
# Stress-ng
35+
RUN apt-get -y install stress-ng
36+
37+
# HTCondor
38+
RUN apt-get install -y curl
39+
RUN curl -fsSL https://get.htcondor.org | sudo /bin/bash -s -- --no-dry-run --channel stable
40+
RUN rm /etc/condor/config.d/00-minicondor
41+
42+
# HTCondor: Create master config
43+
RUN echo "use ROLE: CentralManager" >> /etc/condor/config.d/00-minicondor-master && \
44+
echo "use ROLE: Submit" >> /etc/condor/config.d/00-minicondor-master && \
45+
echo "BIND_ALL_INTERFACES = True" >> /etc/condor/config.d/00-minicondor-master && \
46+
echo "CONDOR_HOST = localhost" >> /etc/condor/config.d/00-minicondor-master && \
47+
echo "SEC_DEFAULT_AUTHENTICATION = OPTIONAL" >> /etc/condor/config.d/00-minicondor-master && \
48+
echo "SEC_DEFAULT_AUTHENTICATION_METHODS = CLAIMTOBE" >> /etc/condor/config.d/00-minicondor-master && \
49+
echo "SCHEDD_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-master && \
50+
echo "NEGOTIATOR_INTERVAL = 2" >> /etc/condor/config.d/00-minicondor-master && \
51+
echo "NEGOTIATOR_CYCLE_DELAY = 5" >> /etc/condor/config.d/00-minicondor-master && \
52+
echo "STARTER_UPDATE_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-master && \
53+
echo "SHADOW_QUEUE_UPDATE_INTERVAL = 10" >> /etc/condor/config.d/00-minicondor-master && \
54+
echo "UPDATE_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-master && \
55+
echo "RUNBENCHMARKS = 0" >> /etc/condor/config.d/00-minicondor-master && \
56+
echo "HOSTALLOW_READ = *" >> /etc/condor/config.d/00-minicondor-master && \
57+
echo "ALLOW_WRITE = *" >> /etc/condor/config.d/00-minicondor-master && \
58+
echo "ALLOW_READ = *" >> /etc/condor/config.d/00-minicondor-master && \
59+
echo "ALLOW_NEGOTIATOR = *" >> /etc/condor/config.d/00-minicondor-master && \
60+
echo "ALLOW_DAEMON = *" >> /etc/condor/config.d/00-minicondor-master && \
61+
echo "ALLOW_COLLECTOR = *" >> /etc/condor/config.d/00-minicondor-master && \
62+
echo "DAGMAN_USE_STRICT = 0" >> /etc/condor/config.d/00-minicondor-master && \
63+
echo "MAX_TRANSFER_OUTPUT_MB = 2048" >> /etc/condor/config.d/00-minicondor-master && \
64+
echo "JOB_MAX_FILE_TRANSFER_SIZE = 100" >> /etc/condor/config.d/00-minicondor-master && \
65+
echo "JOB_DEFAULT_REQUESTDISK = 10MB" >> /etc/condor/config.d/00-minicondor-master && \
66+
echo "NUM_CPUS = 1" >> /etc/condor/config.d/00-minicondor-master
67+
68+
# HTCondor: Create worker config
69+
RUN echo "use ROLE: Execute" >> /etc/condor/config.d/00-minicondor-worker && \
70+
echo "BIND_ALL_INTERFACES = True" >> /etc/condor/config.d/00-minicondor-worker && \
71+
echo "CONDOR_HOST = localhost" >> /etc/condor/config.d/00-minicondor-worker && \
72+
echo "COLLECTOR_HOST = localhost" >> /etc/condor/config.d/00-minicondor-worker && \
73+
echo "SEC_DEFAULT_AUTHENTICATION = OPTIONAL" >> /etc/condor/config.d/00-minicondor-worker && \
74+
echo "SEC_DEFAULT_AUTHENTICATION_METHODS = CLAIMTOBE" >> /etc/condor/config.d/00-minicondor-worker && \
75+
echo "SCHEDD_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-worker && \
76+
echo "NEGOTIATOR_INTERVAL = 2" >> /etc/condor/config.d/00-minicondor-worker && \
77+
echo "NEGOTIATOR_CYCLE_DELAY = 5" >> /etc/condor/config.d/00-minicondor-worker && \
78+
echo "STARTER_UPDATE_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-worker && \
79+
echo "SHADOW_QUEUE_UPDATE_INTERVAL = 10" >> /etc/condor/config.d/00-minicondor-worker && \
80+
echo "UPDATE_INTERVAL = 5" >> /etc/condor/config.d/00-minicondor-worker && \
81+
echo "RUNBENCHMARKS = 0" >> /etc/condor/config.d/00-minicondor-worker && \
82+
echo "HOSTALLOW_WRITE = *" >> /etc/condor/config.d/00-minicondor-worker && \
83+
echo "HOSTALLOW_READ = *" >> /etc/condor/config.d/00-minicondor-worker && \
84+
echo "ALLOW_WRITE = *" >> /etc/condor/config.d/00-minicondor-worker && \
85+
echo "ALLOW_READ = *" >> /etc/condor/config.d/00-minicondor-worker && \
86+
echo "ALLOW_NEGOTIATOR = *" >> /etc/condor/config.d/00-minicondor-worker && \
87+
echo "ALLOW_DAEMON = *" >> /etc/condor/config.d/00-minicondor-worker && \
88+
echo "ALLOW_COLLECTOR = *" >> /etc/condor/config.d/00-minicondor-worker && \
89+
echo "NUM_CPUS = 1" >> /etc/condor/config.d/00-minicondor-worker
90+
91+
# HTCondor: Some setup
92+
RUN mkdir -p /var/lib/condor
93+
RUN mkdir -p /var/lib/condor/log
94+
RUN mkdir -p /var/lib/condor/execute
95+
RUN chown condor:condor -R /var/lib/condor
96+
RUN chmod -R 777 /var/lib/condor/execute
97+
98+
# Install Pegasus
99+
RUN wget -O - http://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
100+
RUN echo 'deb [arch=amd64] http://download.pegasus.isi.edu/pegasus/ubuntu bionic main' | sudo tee /etc/apt/sources.list.d/pegasus.list
101+
RUN apt-get update
102+
RUN apt-get install -y pegasus
103+
RUN python3 -m pip install pegasus-wms.api --break-system-packages
104+
105+
# Add wfcommons user
106+
RUN useradd -ms /bin/bash wfcommons
107+
RUN adduser wfcommons sudo
108+
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
109+
ENV PATH="$PATH:/home/wfcommons/.local/bin/"
110+
111+
USER wfcommons
112+
WORKDIR /home/wfcommons
113+
114+
# Create script to start HTCondor
115+
RUN echo "#!/bin/bash" >> ./start_condor.sh && \
116+
echo "export CONDOR_HOST=localhost" >> ./start_condor.sh && \
117+
echo "export COLLECTOR_HOST=localhost" >> ./start_condor.sh && \
118+
echo "CONDOR_CONFIG=/etc/condor/config.d/00-minicondor-master sudo condor_master" >> ./start_condor.sh && \
119+
echo "sudo condor_collector &" >> ./start_condor.sh && \
120+
echo "CONDOR_CONFIG=/etc/condor/config.d/00-minicondor-worker sudo condor_startd &" >> ./start_condor.sh && \
121+
echo "sleep 3" >> ./start_condor.sh && \
122+
echo "condor_status" >> ./start_condor.sh
123+
RUN chmod +x ./start_condor.sh
124+
125+
# Create script to run the Workflow
126+
RUN echo "#!/bin/bash" >> ./run_workflow.sh && \
127+
echo "pegasus-plan --dir work --cleanup none --output-site local --submit \`ls *.yml\`" >> ./run_workflow.sh && \
128+
echo "sleep 10" >> ./run_workflow.sh && \
129+
echo "PID_FILE=\`ls work/wfcommons/pegasus/*/run0001/*.pid\`" >> ./run_workflow.sh && \
130+
echo "echo \"Waiting for workflow execution to complete...\"" >> ./run_workflow.sh && \
131+
echo "while [[ -f \$PID_FILE ]]" >> ./run_workflow.sh && \
132+
echo "do" >> ./run_workflow.sh && \
133+
echo " echo \"sleeping 5...\"" >> ./run_workflow.sh && \
134+
echo " sleep 5" >> ./run_workflow.sh && \
135+
echo "done" >> ./run_workflow.sh && \
136+
echo "# Sleep 10 so that status in database is up to date..." >> ./run_workflow.sh && \
137+
echo "sleep 10" >> ./run_workflow.sh && \
138+
echo "echo \"Workflow execution complete!\"" >> ./run_workflow.sh
139+
RUN chmod +x ./run_workflow.sh
140+
141+
ENV CONDOR_HOST localhost
142+
ENV COLLECTOR_HOST localhost
143+
144+
145+
146+

‎tests/translators/test_translators.py‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
from wfcommons.wfbench import BashTranslator
3030
from wfcommons.wfbench import TaskVineTranslator
3131
from wfcommons.wfbench import CWLTranslator
32+
from wfcommons.wfbench import PegasusTranslator
3233

3334

3435
def _start_docker_container(backend, mounted_dir, working_dir, bin_dir, command=["sleep", "infinity"]):
@@ -97,7 +98,7 @@ def _create_workflow_benchmark():
9798
benchmark_full_path = "/tmp/blast-benchmark-{desired_num_tasks}.json"
9899
shutil.rmtree(benchmark_full_path, ignore_errors=True)
99100
benchmark = WorkflowBenchmark(recipe=BlastRecipe, num_tasks=desired_num_tasks)
100-
benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=10, data=10, percent_cpu=0.6)
101+
benchmark.create_benchmark(pathlib.Path("/tmp/"), cpu_work=1, data=1, percent_cpu=0.6)
101102
with open(f"/tmp/blast-benchmark-{desired_num_tasks}.json", "r") as f:
102103
generated_json = json.load(f)
103104
num_tasks = len(generated_json["workflow"]["specification"]["tasks"])
@@ -120,6 +121,16 @@ def _additional_setup_taskvine(container):
120121
cmd=["bash", "-c", "source ~/conda/etc/profile.d/conda.sh && conda activate && vine_worker localhost 9123"],
121122
detach=True, stdout=True, stderr=True)
122123

124+
def _additional_setup_pegasus(container):
125+
# Start Condor
126+
exit_code, output = container.exec_run(cmd=["bash", "-c",
127+
"bash /home/wfcommons/start_condor.sh"],
128+
stdout=True, stderr=True)
129+
# Run pegasus script
130+
exit_code, output = container.exec_run(cmd=["bash", "-c",
131+
"python3 ./pegasus_workflow.py"],
132+
stdout=True, stderr=True)
133+
123134

124135
additional_setup_methods = {
125136
"dask": noop,
@@ -129,6 +140,7 @@ def _additional_setup_taskvine(container):
129140
"bash": noop,
130141
"taskvine": _additional_setup_taskvine,
131142
"cwl": noop,
143+
"pegasus": _additional_setup_pegasus,
132144
}
133145

134146
#############################################################################
@@ -196,7 +208,6 @@ def run_workflow_taskvine(container, num_tasks, str_dirpath):
196208
assert (output.decode().count("completed") == num_tasks)
197209

198210
def run_workflow_cwl(container, num_tasks, str_dirpath):
199-
200211
# Run the workflow!
201212
# Note that the input file is hardcoded and Blast-specific
202213
exit_code, output = container.exec_run(cmd="cwltool ./main.cwl --split_fasta_00000001_input ./data/workflow_infile_0001 ", stdout=True, stderr=True)
@@ -208,6 +219,16 @@ def run_workflow_cwl(container, num_tasks, str_dirpath):
208219
# and there is a 2* because there is a message for the job and for the step)
209220
assert (output.decode().count("completed success") == 3 + 2 *num_tasks)
210221

222+
def run_workflow_pegasus(container, num_tasks, str_dirpath):
223+
# Run the workflow!
224+
exit_code, output = container.exec_run(cmd="bash /home/wfcommons/run_workflow.sh", stdout=True, stderr=True)
225+
ignored, status_output = container.exec_run(cmd="pegasus-status -l /tmp/pegasus_translated_workflow/work/wfcommons/pegasus/Blast-Benchmark/run0001", stdout=True, stderr=True)
226+
# Kill the container
227+
container.remove(force=True)
228+
# Check sanity
229+
assert(exit_code == 0)
230+
assert("Workflow execution complete!" in output.decode())
231+
assert(status_output.decode().count("Success") == 2)
211232

212233
run_workflow_methods = {
213234
"dask": run_workflow_dask,
@@ -217,6 +238,7 @@ def run_workflow_cwl(container, num_tasks, str_dirpath):
217238
"bash": run_workflow_bash,
218239
"taskvine": run_workflow_taskvine,
219240
"cwl": run_workflow_cwl,
241+
"pegasus": run_workflow_pegasus,
220242
}
221243

222244
translator_classes = {
@@ -227,6 +249,7 @@ def run_workflow_cwl(container, num_tasks, str_dirpath):
227249
"bash": BashTranslator,
228250
"taskvine": TaskVineTranslator,
229251
"cwl": CWLTranslator,
252+
"pegasus": PegasusTranslator,
230253
}
231254

232255

@@ -242,6 +265,7 @@ class TestTranslators:
242265
"bash",
243266
"taskvine",
244267
"cwl",
268+
"pegasus",
245269
])
246270
@pytest.mark.unit
247271
# @pytest.mark.skip(reason="tmp")
@@ -271,4 +295,4 @@ def test_translator(self, backend) -> None:
271295
sys.stderr.write("Running workflow...\n")
272296
start_time = time.time()
273297
run_workflow_methods[backend](container, num_tasks, str_dirpath)
274-
sys.stderr.write("Workflow ran in %.2f seconds\n" % (time.time() - start_time))
298+
sys.stderr.write("Workflow ran in %.2f seconds\n" % (time.time() - start_time))

‎wfcommons/wfbench/translator/pegasus.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,10 @@ def _add_task(self, task_name: str, parent_task: Optional[str] = None, tasks_pri
123123
children = self.task_children[task_name]
124124

125125
# Generate input spec
126-
input_spec = "["
126+
input_spec = "\"["
127127
for f in task.input_files:
128-
input_spec += f"\"{f.file_id}\","
129-
input_spec = input_spec[:-1] + "]"
128+
input_spec += f"\\\\\"{f.file_id}\\\\\","
129+
input_spec = input_spec[:-1] + "]\""
130130

131131
# output files
132132
output_spec = "\"{"

0 commit comments

Comments
 (0)