Skip to content

Commit 42533c7

Browse files
piEspositopiEsposito
committed
Merge remote-tracking branch 'upstream/main' into main
2 parents be73dfb + 0b61cea commit 42533c7

45 files changed

Lines changed: 1446 additions & 150 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build_docker_images.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: Build Docker images (nightly)
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "0 0 * * *" # every day at midnight
7+
8+
concurrency:
9+
group: docker-image-builds
10+
cancel-in-progress: false
11+
12+
env:
13+
REGISTRY: diffusers
14+
15+
jobs:
16+
build-docker-images:
17+
runs-on: ubuntu-latest
18+
19+
permissions:
20+
contents: read
21+
packages: write
22+
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
image-name:
27+
- diffusers-pytorch-cpu
28+
- diffusers-pytorch-cuda
29+
- diffusers-flax-cpu
30+
- diffusers-flax-tpu
31+
- diffusers-onnxruntime-cpu
32+
- diffusers-onnxruntime-cuda
33+
34+
steps:
35+
- name: Checkout repository
36+
uses: actions/checkout@v3
37+
38+
- name: Login to Docker Hub
39+
uses: docker/login-action@v2
40+
with:
41+
username: ${{ env.REGISTRY }}
42+
password: ${{ secrets.DOCKERHUB_TOKEN }}
43+
44+
- name: Build and push
45+
uses: docker/build-push-action@v3
46+
with:
47+
no-cache: true
48+
context: ./docker/${{ matrix.image-name }}
49+
push: true
50+
tags: ${{ env.REGISTRY }}/${{ matrix.image-name }}:latest

‎.github/workflows/pr_tests.yml‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,45 @@ concurrency:
1111

1212
env:
1313
DIFFUSERS_IS_CI: yes
14-
OMP_NUM_THREADS: 8
15-
MKL_NUM_THREADS: 8
14+
OMP_NUM_THREADS: 4
15+
MKL_NUM_THREADS: 4
1616
PYTEST_TIMEOUT: 60
1717
MPS_TORCH_VERSION: 1.13.0
1818

1919
jobs:
20-
run_tests_cpu:
21-
name: CPU tests on Ubuntu
22-
runs-on: [ self-hosted, docker-gpu ]
20+
run_fast_tests:
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
config:
25+
- name: Fast PyTorch CPU tests on Ubuntu
26+
framework: pytorch
27+
runner: docker-cpu
28+
image: diffusers/diffusers-pytorch-cpu
29+
report: torch_cpu
30+
- name: Fast Flax CPU tests on Ubuntu
31+
framework: flax
32+
runner: docker-cpu
33+
image: diffusers/diffusers-flax-cpu
34+
report: flax_cpu
35+
- name: Fast ONNXRuntime CPU tests on Ubuntu
36+
framework: onnxruntime
37+
runner: docker-cpu
38+
image: diffusers/diffusers-onnxruntime-cpu
39+
report: onnx_cpu
40+
41+
name: ${{ matrix.config.name }}
42+
43+
runs-on: ${{ matrix.config.runner }}
44+
2345
container:
24-
image: python:3.7
46+
image: ${{ matrix.config.image }}
2547
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
2648

49+
defaults:
50+
run:
51+
shell: bash
52+
2753
steps:
2854
- name: Checkout diffusers
2955
uses: actions/checkout@v3
@@ -32,34 +58,56 @@ jobs:
3258

3359
- name: Install dependencies
3460
run: |
35-
python -m pip install --upgrade pip
36-
python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu
3761
python -m pip install -e .[quality,test]
3862
python -m pip install git+https://github.com/huggingface/accelerate
3963
4064
- name: Environment
4165
run: |
4266
python utils/print_env.py
4367
44-
- name: Run all fast tests on CPU
68+
- name: Run fast PyTorch CPU tests
69+
if: ${{ matrix.config.framework == 'pytorch' }}
70+
env:
71+
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
72+
run: |
73+
python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \
74+
-s -v -k "not Flax and not Onnx" \
75+
--make-reports=tests_${{ matrix.config.report }} \
76+
tests/
77+
78+
- name: Run fast Flax TPU tests
79+
if: ${{ matrix.config.framework == 'flax' }}
80+
env:
81+
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
82+
run: |
83+
python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \
84+
-s -v -k "Flax" \
85+
--make-reports=tests_${{ matrix.config.report }} \
86+
tests/
87+
88+
- name: Run fast ONNXRuntime CPU tests
89+
if: ${{ matrix.config.framework == 'onnxruntime' }}
4590
env:
4691
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
4792
run: |
48-
python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/
93+
python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile \
94+
-s -v -k "Onnx" \
95+
--make-reports=tests_${{ matrix.config.report }} \
96+
tests/
4997
5098
- name: Failure short reports
5199
if: ${{ failure() }}
52-
run: cat reports/tests_torch_cpu_failures_short.txt
100+
run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt
53101

54102
- name: Test suite reports artifacts
55103
if: ${{ always() }}
56104
uses: actions/upload-artifact@v2
57105
with:
58-
name: pr_torch_cpu_test_reports
106+
name: pr_${{ matrix.config.report }}_test_reports
59107
path: reports
60108

61-
run_tests_apple_m1:
62-
name: MPS tests on Apple M1
109+
run_fast_tests_apple_m1:
110+
name: Fast PyTorch MPS tests on MacOS
63111
runs-on: [ self-hosted, apple-m1 ]
64112

65113
steps:
@@ -91,7 +139,7 @@ jobs:
91139
run: |
92140
${CONDA_RUN} python utils/print_env.py
93141
94-
- name: Run all fast tests on MPS
142+
- name: Run fast PyTorch tests on M1 (MPS)
95143
shell: arch -arch arm64 bash {0}
96144
env:
97145
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}

‎.github/workflows/push_tests.yml‎

Lines changed: 69 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,38 @@ env:
1414
RUN_SLOW: yes
1515

1616
jobs:
17-
run_tests_single_gpu:
18-
name: Diffusers tests
19-
runs-on: [ self-hosted, docker-gpu, single-gpu ]
17+
run_slow_tests:
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
config:
22+
- name: Slow PyTorch CUDA tests on Ubuntu
23+
framework: pytorch
24+
runner: docker-gpu
25+
image: diffusers/diffusers-pytorch-cuda
26+
report: torch_cuda
27+
- name: Slow Flax TPU tests on Ubuntu
28+
framework: flax
29+
runner: docker-tpu
30+
image: diffusers/diffusers-flax-tpu
31+
report: flax_tpu
32+
- name: Slow ONNXRuntime CUDA tests on Ubuntu
33+
framework: onnxruntime
34+
runner: docker-gpu
35+
image: diffusers/diffusers-onnxruntime-cuda
36+
report: onnx_cuda
37+
38+
name: ${{ matrix.config.name }}
39+
40+
runs-on: ${{ matrix.config.runner }}
41+
2042
container:
21-
image: nvcr.io/nvidia/pytorch:22.07-py3
22-
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache
43+
image: ${{ matrix.config.image }}
44+
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ ${{ matrix.config.runner == 'docker-tpu' && '--privileged' || '--gpus 0'}}
45+
46+
defaults:
47+
run:
48+
shell: bash
2349

2450
steps:
2551
- name: Checkout diffusers
@@ -28,44 +54,68 @@ jobs:
2854
fetch-depth: 2
2955

3056
- name: NVIDIA-SMI
57+
if : ${{ matrix.config.runner == 'docker-gpu' }}
3158
run: |
3259
nvidia-smi
3360
3461
- name: Install dependencies
3562
run: |
36-
python -m pip install --upgrade pip
37-
python -m pip uninstall -y torch torchvision torchtext
38-
python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu117
3963
python -m pip install -e .[quality,test]
4064
python -m pip install git+https://github.com/huggingface/accelerate
4165
4266
- name: Environment
4367
run: |
4468
python utils/print_env.py
4569
46-
- name: Run all (incl. slow) tests on GPU
70+
- name: Run slow PyTorch CUDA tests
71+
if: ${{ matrix.config.framework == 'pytorch' }}
72+
env:
73+
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
74+
run: |
75+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
76+
-s -v -k "not Flax and not Onnx" \
77+
--make-reports=tests_${{ matrix.config.report }} \
78+
tests/
79+
80+
- name: Run slow Flax TPU tests
81+
if: ${{ matrix.config.framework == 'flax' }}
4782
env:
4883
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
4984
run: |
50-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_gpu tests/
85+
python -m pytest -n 0 \
86+
-s -v -k "Flax" \
87+
--make-reports=tests_${{ matrix.config.report }} \
88+
tests/
89+
90+
- name: Run slow ONNXRuntime CUDA tests
91+
if: ${{ matrix.config.framework == 'onnxruntime' }}
92+
env:
93+
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
94+
run: |
95+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
96+
-s -v -k "Onnx" \
97+
--make-reports=tests_${{ matrix.config.report }} \
98+
tests/
5199
52100
- name: Failure short reports
53101
if: ${{ failure() }}
54-
run: cat reports/tests_torch_gpu_failures_short.txt
102+
run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt
55103

56104
- name: Test suite reports artifacts
57105
if: ${{ always() }}
58106
uses: actions/upload-artifact@v2
59107
with:
60-
name: torch_test_reports
108+
name: ${{ matrix.config.report }}_test_reports
61109
path: reports
62110

63-
run_examples_single_gpu:
64-
name: Examples tests
65-
runs-on: [ self-hosted, docker-gpu, single-gpu ]
111+
run_examples_tests:
112+
name: Examples PyTorch CUDA tests on Ubuntu
113+
114+
runs-on: docker-gpu
115+
66116
container:
67-
image: nvcr.io/nvidia/pytorch:22.07-py3
68-
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache
117+
image: diffusers/diffusers-pytorch-cuda
118+
options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
69119

70120
steps:
71121
- name: Checkout diffusers
@@ -79,9 +129,6 @@ jobs:
79129
80130
- name: Install dependencies
81131
run: |
82-
python -m pip install --upgrade pip
83-
python -m pip uninstall -y torch torchvision torchtext
84-
python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cu117
85132
python -m pip install -e .[quality,test,training]
86133
python -m pip install git+https://github.com/huggingface/accelerate
87134
@@ -93,11 +140,11 @@ jobs:
93140
env:
94141
HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
95142
run: |
96-
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_gpu examples/
143+
python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=examples_torch_cuda examples/
97144
98145
- name: Failure short reports
99146
if: ${{ failure() }}
100-
run: cat reports/examples_torch_gpu_failures_short.txt
147+
run: cat reports/examples_torch_cuda_failures_short.txt
101148

102149
- name: Test suite reports artifacts
103150
if: ${{ always() }}

‎docker/diffusers-flax-cpu/Dockerfile‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
FROM ubuntu:20.04
2+
LABEL maintainer="Hugging Face"
3+
LABEL repository="diffusers"
4+
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
RUN apt update && \
8+
apt install -y bash \
9+
build-essential \
10+
git \
11+
git-lfs \
12+
curl \
13+
ca-certificates \
14+
python3.8 \
15+
python3-pip \
16+
python3.8-venv && \
17+
rm -rf /var/lib/apt/lists
18+
19+
# make sure to use venv
20+
RUN python3 -m venv /opt/venv
21+
ENV PATH="/opt/venv/bin:$PATH"
22+
23+
# pre-install the heavy dependencies (these can later be overridden by the deps from setup.py)
24+
# follow the instructions here: https://cloud.google.com/tpu/docs/run-in-container#train_a_jax_model_in_a_docker_container
25+
RUN python3 -m pip install --no-cache-dir --upgrade pip && \
26+
python3 -m pip install --upgrade --no-cache-dir \
27+
clu \
28+
"jax[cpu]>=0.2.16,!=0.3.2" \
29+
"flax>=0.4.1" \
30+
"jaxlib>=0.1.65" && \
31+
python3 -m pip install --no-cache-dir \
32+
accelerate \
33+
datasets \
34+
hf-doc-builder \
35+
huggingface-hub \
36+
modelcards \
37+
numpy \
38+
scipy \
39+
tensorboard \
40+
transformers
41+
42+
CMD ["/bin/bash"]

0 commit comments

Comments
 (0)