Skip to content

speedup with Intel Ipex cpu compilation with default torchserve handlers and config changes #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rbavery wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

speedup with Intel Ipex cpu compilation with default torchserve handlers and config changes #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49b57d6
torchscript with Ipex flags set, unsure if used, 6 sec per inf on cpu
rbavery Apr 26, 2023
d6f5df0
add note on intel ipex vs onnx and update prep steps
rbavery Apr 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/megadetectorv5/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ USER root
RUN pip install "gitpython" "ipython" "matplotlib>=3.2.2" "numpy==1.23.4" "opencv-python==4.6.0.66" \
"Pillow==9.2.0" "psutil" "PyYAML>=5.3.1" "requests>=2.23.0" "scipy==1.9.3" "thop>=0.1.1" \
"torch==1.10.0" "torchvision==0.11.1" "tqdm>=4.64.0" "tensorboard>=2.4.1" "pandas>=1.1.4" \
"seaborn>=0.11.0" "setuptools>=65.5.1" "onnxruntime==1.14.1" "onnx==1.13.1"
"seaborn>=0.11.0" "setuptools>=65.5.1" "intel-extension-for-pytorch"
COPY ./deployment/dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh
RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh
RUN mkdir -p /home/model-server/ && mkdir -p /home/model-server/tmp
Expand All @@ -16,5 +16,6 @@ WORKDIR /home/model-server
ENV TEMP=/home/model-server/tmp
ENV YOLOv5_AUTOINSTALL=False
ENV ENABLE_TORCH_PROFILER=TRUE
ENV TS_IPEX_ENABLE=TRUE
ENTRYPOINT ["/usr/local/bin/dockerd-entrypoint.sh"]
CMD ["serve"]
9 changes: 5 additions & 4 deletions api/megadetectorv5/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ From this directory, run:
```
aws s3 sync s3://animl-model-zoo/mdv5-weights-models/ model-weights
```
TODO revise readme to cover using torchscript + Intel IPEX if it works out faster than ONNX

If you want to quickly run model inference outside of the deployment environment without post processing steps, you can use the ONNX model file.

Expand All @@ -35,13 +36,13 @@ conda activate mdv5a
pip install "gitpython" "ipython" "matplotlib>=3.2.2" "numpy==1.23.4" "opencv-python==4.6.0.66" \
"Pillow==9.2.0" "psutil" "PyYAML>=5.3.1" "requests>=2.23.0" "scipy==1.9.3" "thop>=0.1.1" \
"torch==1.10.0" "torchvision==0.11.1" "tqdm>=4.64.0" "tensorboard>=2.4.1" "pandas>=1.1.4" \
"seaborn>=0.11.0" "setuptools>=65.5.1" "onnxruntime==1.14.1" "onnx==1.13.1", "torch-model-archiver", "httpx"
"seaborn>=0.11.0" "setuptools>=65.5.1" "intel-extension-for-pytorch", "torch-model-archiver", "httpx"
```
then, run the export step

```
python yolov5/export.py --imgsz '(960,1280)' --weights model-weights/md_v5a.0.0.pt --include onnx
mv model-weights/md_v5a.0.0.onnx model-weights/md_v5a.0.0.960.1280.onnx
python yolov5/export.py --imgsz '(960,1280)' --weights model-weights/md_v5a.0.0.pt --include torchscript
mv model-weights/md_v5a.0.0.torchscript model-weights/md_v5a.0.0.960.1280.torchscript
```

Note, we are using the yolov5 source when compiling the model for deployment. This is [yolov5 commit hash 5c91da](https://github.com/ultralytics/yolov5/tree/5c91daeaecaeca709b8b6d13bd571d068fdbd003)
Expand All @@ -55,7 +56,7 @@ this will create models/megadetectorv5/md_v5a.0.0.onnx and move it to the correc
`pip install torch-model-archiver` then,

```
torch-model-archiver --model-name mdv5a --version 1.0.0 --serialized-file model-weights/md_v5a.0.0.960.1280.onnx --extra-files index_to_name.json --handler mdv5_handler.py
torch-model-archiver --model-name mdv5a --version 1.0.0 --serialized-file model-weights/md_v5a.0.0.960.1280.torchscript --extra-files index_to_name.json --handler mdv5_handler.py
mv mdv5a.mar model_store/mdv5a.mar
```

Expand Down
62 changes: 45 additions & 17 deletions api/megadetectorv5/debug_single_img_inference.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "b6be1190-1992-4a43-a2af-dd1c9438caec",
"metadata": {
"tags": []
Expand Down Expand Up @@ -576,8 +576,49 @@
},
{
"cell_type": "code",
"execution_count": 14,
"id": "db8a037a-3737-4423-ae9f-fa7721e768be",
"execution_count": null,
"id": "c8c7ab74-197e-4960-a6f6-628727d7d55c",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 6,
"id": "01c6b399-d721-46ff-8e4c-3d54014b3669",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"url = \"http://127.0.0.1:8080/predictions/mdv5a\"\n",
"headers = {\n",
" \"Content-Type\": \"application/octet-stream\",\n",
"}\n",
"\n",
"ts_top_results = {}\n",
"# pth= \"../../input/sample-img-fox.jpg\"\n",
"pth = '../../input/coolpics/IMG_0314.JPG'\n",
"with open(pth, \"rb\") as f:\n",
" image_data = f.read()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "42377845-7c1f-4a64-a11c-23a846fd533b",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"response = httpx.post(url, data=image_data, headers=headers, timeout=10)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "36b9b252-d03f-4875-987e-4eef1bf50f83",
"metadata": {
"tags": []
},
Expand All @@ -589,25 +630,12 @@
"<PIL.MpoImagePlugin.MpoImageFile image mode=RGB size=4208x2368>"
]
},
"execution_count": 14,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"url = \"http://127.0.0.1:8080/predictions/mdv5a\"\n",
"headers = {\n",
" \"Content-Type\": \"application/octet-stream\",\n",
"}\n",
"\n",
"ts_top_results = {}\n",
"# pth= \"../../input/sample-img-fox.jpg\"\n",
"pth = '../../input/coolpics/IMG_0314.JPG'\n",
"with open(pth, \"rb\") as f:\n",
" image_data = f.read()\n",
"# time.sleep(5)\n",
"response = httpx.post(url, data=image_data, headers=headers)\n",
"\n",
"if response.status_code == 200:\n",
" # do something with the response\n",
" ts_top_results.update({pth.split(\"/\")[-1]:response.json()[0]})\n",
Expand Down
78 changes: 39 additions & 39 deletions api/megadetectorv5/mdv5_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
import cv2
import base64
import torch
import onnx
import onnxruntime as ort
# import onnx
# import onnxruntime as ort
import io
import torchvision
import torch
Expand Down Expand Up @@ -81,42 +81,42 @@ def preprocess(self, data):
# has shape BATCH_SIZE=1 x 3 x IMG_SIZE x IMG_SIZE
return image

def initialize(self, context):
"""
Invoke by torchserve for loading a model
:param context: context contains model server system properties
:return:
"""
start = time()
# load the model
self.manifest = context.manifest
properties = context.system_properties
model_dir = properties.get("model_dir")
self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
# Read onnx file
serialized_file = self.manifest['model']['serializedFile']
model_path = os.path.join(model_dir, serialized_file)
# Model
self.ort_session = ort.InferenceSession(model_path)
self.initialized = True
print("XXXXX Initialization time: ", time()-start)

def inference(self, model_input):
"""
Internal inference methods
:param model_input: transformed model input data
:return: list of inference output in NDArray
"""
start = time()
# Do some inference call to engine here and return output
model_output = self.ort_session.run(
None,
{"images": model_input.numpy().astype(np.float32)},
)
print("XXXXX Inference time: ", time()-start)
print(len(model_output))
print(type(model_output))
return torch.Tensor(model_output)
# def initialize(self, context):
# """
# Invoke by torchserve for loading a model
# :param context: context contains model server system properties
# :return:
# """
# start = time()
# # load the model
# self.manifest = context.manifest
# properties = context.system_properties
# model_dir = properties.get("model_dir")
# self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu")
# # Read onnx file
# serialized_file = self.manifest['model']['serializedFile']
# model_path = os.path.join(model_dir, serialized_file)
# # Model
# self.ort_session = ort.InferenceSession(model_path)
# self.initialized = True
# print("XXXXX Initialization time: ", time()-start)

# def inference(self, model_input):
# """
# Internal inference methods
# :param model_input: transformed model input data
# :return: list of inference output in NDArray
# """
# start = time()
# # Do some inference call to engine here and return output
# model_output = self.ort_session.run(
# None,
# {"images": model_input.numpy().astype(np.float32)},
# )
# print("XXXXX Inference time: ", time()-start)
# print(len(model_output))
# print(type(model_output))
# return torch.Tensor(model_output)


def postprocess(self, inference_output):
Expand Down Expand Up @@ -169,6 +169,7 @@ def handle(self, data, context):
"""
model_input = self.preprocess(data)
model_output = self.inference(model_input)
model_output = model_output[0] # for some reason the ipex model returns tuple with only one element
return self.postprocess(model_output)


Expand All @@ -178,7 +179,6 @@ def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=Non
Returns:
list of detections, on (n,6) tensor per image [xyxy, conf, cls]
"""

nc = prediction.shape[2] - 5 # number of classes
xc = prediction[..., 4] > conf_thres # candidates

Expand Down