diff --git a/api/megadetectorv5/Dockerfile b/api/megadetectorv5/Dockerfile index f697b04..caee17a 100644 --- a/api/megadetectorv5/Dockerfile +++ b/api/megadetectorv5/Dockerfile @@ -7,7 +7,7 @@ USER root RUN pip install "gitpython" "ipython" "matplotlib>=3.2.2" "numpy==1.23.4" "opencv-python==4.6.0.66" \ "Pillow==9.2.0" "psutil" "PyYAML>=5.3.1" "requests>=2.23.0" "scipy==1.9.3" "thop>=0.1.1" \ "torch==1.10.0" "torchvision==0.11.1" "tqdm>=4.64.0" "tensorboard>=2.4.1" "pandas>=1.1.4" \ - "seaborn>=0.11.0" "setuptools>=65.5.1" "onnxruntime==1.14.1" "onnx==1.13.1" + "seaborn>=0.11.0" "setuptools>=65.5.1" "intel-extension-for-pytorch" COPY ./deployment/dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh RUN mkdir -p /home/model-server/ && mkdir -p /home/model-server/tmp @@ -16,5 +16,6 @@ WORKDIR /home/model-server ENV TEMP=/home/model-server/tmp ENV YOLOv5_AUTOINSTALL=False ENV ENABLE_TORCH_PROFILER=TRUE +ENV TS_IPEX_ENABLE=TRUE ENTRYPOINT ["/usr/local/bin/dockerd-entrypoint.sh"] CMD ["serve"] \ No newline at end of file diff --git a/api/megadetectorv5/README.md b/api/megadetectorv5/README.md index 2c0f9d4..028901e 100644 --- a/api/megadetectorv5/README.md +++ b/api/megadetectorv5/README.md @@ -20,6 +20,7 @@ From this directory, run: ``` aws s3 sync s3://animl-model-zoo/mdv5-weights-models/ model-weights ``` +TODO revise readme to cover using torchscript + Intel IPEX if it works out faster than ONNX If you want to quickly run model inference outside of the deployment environment without post processing steps, you can use the ONNX model file. @@ -35,13 +36,13 @@ conda activate mdv5a pip install "gitpython" "ipython" "matplotlib>=3.2.2" "numpy==1.23.4" "opencv-python==4.6.0.66" \ "Pillow==9.2.0" "psutil" "PyYAML>=5.3.1" "requests>=2.23.0" "scipy==1.9.3" "thop>=0.1.1" \ "torch==1.10.0" "torchvision==0.11.1" "tqdm>=4.64.0" "tensorboard>=2.4.1" "pandas>=1.1.4" \ -"seaborn>=0.11.0" "setuptools>=65.5.1" "onnxruntime==1.14.1" "onnx==1.13.1", "torch-model-archiver", "httpx" +"seaborn>=0.11.0" "setuptools>=65.5.1" "intel-extension-for-pytorch", "torch-model-archiver", "httpx" ``` then, run the export step ``` -python yolov5/export.py --imgsz '(960,1280)' --weights model-weights/md_v5a.0.0.pt --include onnx -mv model-weights/md_v5a.0.0.onnx model-weights/md_v5a.0.0.960.1280.onnx +python yolov5/export.py --imgsz '(960,1280)' --weights model-weights/md_v5a.0.0.pt --include torchscript +mv model-weights/md_v5a.0.0.torchscript model-weights/md_v5a.0.0.960.1280.torchscript ``` Note, we are using the yolov5 source when compiling the model for deployment. This is [yolov5 commit hash 5c91da](https://github.com/ultralytics/yolov5/tree/5c91daeaecaeca709b8b6d13bd571d068fdbd003) @@ -55,7 +56,7 @@ this will create models/megadetectorv5/md_v5a.0.0.onnx and move it to the correc `pip install torch-model-archiver` then, ``` -torch-model-archiver --model-name mdv5a --version 1.0.0 --serialized-file model-weights/md_v5a.0.0.960.1280.onnx --extra-files index_to_name.json --handler mdv5_handler.py +torch-model-archiver --model-name mdv5a --version 1.0.0 --serialized-file model-weights/md_v5a.0.0.960.1280.torchscript --extra-files index_to_name.json --handler mdv5_handler.py mv mdv5a.mar model_store/mdv5a.mar ``` diff --git a/api/megadetectorv5/debug_single_img_inference.ipynb b/api/megadetectorv5/debug_single_img_inference.ipynb index 5356b8e..fc73a06 100644 --- a/api/megadetectorv5/debug_single_img_inference.ipynb +++ b/api/megadetectorv5/debug_single_img_inference.ipynb @@ -376,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "b6be1190-1992-4a43-a2af-dd1c9438caec", "metadata": { "tags": [] @@ -576,8 +576,49 @@ }, { "cell_type": "code", - "execution_count": 14, - "id": "db8a037a-3737-4423-ae9f-fa7721e768be", + "execution_count": null, + "id": "c8c7ab74-197e-4960-a6f6-628727d7d55c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "01c6b399-d721-46ff-8e4c-3d54014b3669", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "url = \"http://127.0.0.1:8080/predictions/mdv5a\"\n", + "headers = {\n", + " \"Content-Type\": \"application/octet-stream\",\n", + "}\n", + "\n", + "ts_top_results = {}\n", + "# pth= \"../../input/sample-img-fox.jpg\"\n", + "pth = '../../input/coolpics/IMG_0314.JPG'\n", + "with open(pth, \"rb\") as f:\n", + " image_data = f.read()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "42377845-7c1f-4a64-a11c-23a846fd533b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "response = httpx.post(url, data=image_data, headers=headers, timeout=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "36b9b252-d03f-4875-987e-4eef1bf50f83", "metadata": { "tags": [] }, @@ -589,25 +630,12 @@ "" ] }, - "execution_count": 14, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "url = \"http://127.0.0.1:8080/predictions/mdv5a\"\n", - "headers = {\n", - " \"Content-Type\": \"application/octet-stream\",\n", - "}\n", - "\n", - "ts_top_results = {}\n", - "# pth= \"../../input/sample-img-fox.jpg\"\n", - "pth = '../../input/coolpics/IMG_0314.JPG'\n", - "with open(pth, \"rb\") as f:\n", - " image_data = f.read()\n", - "# time.sleep(5)\n", - "response = httpx.post(url, data=image_data, headers=headers)\n", - "\n", "if response.status_code == 200:\n", " # do something with the response\n", " ts_top_results.update({pth.split(\"/\")[-1]:response.json()[0]})\n", diff --git a/api/megadetectorv5/mdv5_handler.py b/api/megadetectorv5/mdv5_handler.py index 3872b6e..1230bab 100644 --- a/api/megadetectorv5/mdv5_handler.py +++ b/api/megadetectorv5/mdv5_handler.py @@ -5,8 +5,8 @@ import cv2 import base64 import torch -import onnx -import onnxruntime as ort +# import onnx +# import onnxruntime as ort import io import torchvision import torch @@ -81,42 +81,42 @@ def preprocess(self, data): # has shape BATCH_SIZE=1 x 3 x IMG_SIZE x IMG_SIZE return image - def initialize(self, context): - """ - Invoke by torchserve for loading a model - :param context: context contains model server system properties - :return: - """ - start = time() - # load the model - self.manifest = context.manifest - properties = context.system_properties - model_dir = properties.get("model_dir") - self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu") - # Read onnx file - serialized_file = self.manifest['model']['serializedFile'] - model_path = os.path.join(model_dir, serialized_file) - # Model - self.ort_session = ort.InferenceSession(model_path) - self.initialized = True - print("XXXXX Initialization time: ", time()-start) - - def inference(self, model_input): - """ - Internal inference methods - :param model_input: transformed model input data - :return: list of inference output in NDArray - """ - start = time() - # Do some inference call to engine here and return output - model_output = self.ort_session.run( - None, - {"images": model_input.numpy().astype(np.float32)}, - ) - print("XXXXX Inference time: ", time()-start) - print(len(model_output)) - print(type(model_output)) - return torch.Tensor(model_output) + # def initialize(self, context): + # """ + # Invoke by torchserve for loading a model + # :param context: context contains model server system properties + # :return: + # """ + # start = time() + # # load the model + # self.manifest = context.manifest + # properties = context.system_properties + # model_dir = properties.get("model_dir") + # self.device = torch.device("cuda:" + str(properties.get("gpu_id")) if torch.cuda.is_available() else "cpu") + # # Read onnx file + # serialized_file = self.manifest['model']['serializedFile'] + # model_path = os.path.join(model_dir, serialized_file) + # # Model + # self.ort_session = ort.InferenceSession(model_path) + # self.initialized = True + # print("XXXXX Initialization time: ", time()-start) + + # def inference(self, model_input): + # """ + # Internal inference methods + # :param model_input: transformed model input data + # :return: list of inference output in NDArray + # """ + # start = time() + # # Do some inference call to engine here and return output + # model_output = self.ort_session.run( + # None, + # {"images": model_input.numpy().astype(np.float32)}, + # ) + # print("XXXXX Inference time: ", time()-start) + # print(len(model_output)) + # print(type(model_output)) + # return torch.Tensor(model_output) def postprocess(self, inference_output): @@ -169,6 +169,7 @@ def handle(self, data, context): """ model_input = self.preprocess(data) model_output = self.inference(model_input) + model_output = model_output[0] # for some reason the ipex model returns tuple with only one element return self.postprocess(model_output) @@ -178,7 +179,6 @@ def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=Non Returns: list of detections, on (n,6) tensor per image [xyxy, conf, cls] """ - nc = prediction.shape[2] - 5 # number of classes xc = prediction[..., 4] > conf_thres # candidates