-
Notifications
You must be signed in to change notification settings - Fork 78
Open
Description
--gpu-id -1 is broken:
~/repos/iree-turbine$ ROCR_VISIBLE_DEVICES=0,1 iree-boo-driver conv --gpu-id -1
>>> conv
Traceback (most recent call last):
File "/home/rkayaith/repos/iree-turbine/iree/turbine/kernel/boo/driver/driver.py", line 188, in main
prof = run(
^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/kernel/boo/driver/driver.py", line 285, in run
example_results = func(*per_device_args[0])
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/kernel/boo/exports/signature.py", line 76, in run_and_verify
result = compiled(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 414, in __call__
return super().__call__(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 832, in compile_wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/kernel/boo/op_exports/conv.py", line 442, in forward
def forward(self, *args: torch.Tensor) -> torch.Tensor:
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/aot_autograd.py", line 1130, in forward
return compiled_fn(full_args)
^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 353, in runtime_wrapper
all_outs = call_func_at_runtime_with_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 129, in call_func_at_runtime_with_args
out = normalize_as_list(f(args))
^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 724, in inner_fn
outs = compiled_fn(args)
^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 526, in wrapper
return compiled_fn(runtime_args)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_functorch/_aot_autograd/utils.py", line 103, in g
return f(*args)
^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/fx/_lazy_graph_module.py", line 126, in _lazy_forward
return self(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/fx/graph_module.py", line 837, in call_wrapped
return self._wrapped_call(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/fx/graph_module.py", line 413, in __call__
raise e
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/fx/graph_module.py", line 400, in __call__
return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<eval_with_key>.5 from /home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/fx/experimental/proxy_tensor.py:1350 in wrapped", line 7, in forward
fused_op_layout_cus_b2e5e1ff62d0dafe94e32d1e47e3fd273cc15dd5 = torch.ops.boo.fused_op_layout_cus_b2e5e1ff62d0dafe94e32d1e47e3fd273cc15dd5(permute, permute_1); permute = permute_1 = None
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_ops.py", line 1255, in __call__
return self._op(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/kernel/boo/ops/graph.py", line 314, in _
outputs = launch(*[arg.data for arg in handled_inputs])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/runtime/launch.py", line 230, in __call__
arg_list.push_ref(turbine_device.import_torch_tensor(arg))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/runtime/device.py", line 405, in <lambda>
self.import_torch_tensor = lambda t: import_fn(self, t)
^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/iree-turbine/iree/turbine/runtime/device.py", line 551, in _device_import_torch_tensor_cuda_hip
capsule = t.__dlpack__(stream=None)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/rkayaith/repos/.venv/lib/python3.11/site-packages/torch/_tensor.py", line 1726, in __dlpack__
raise BufferError(
BufferError: Can't export tensors on a different CUDA device index. Expected: 0. Current device: 1.I believe this is related to the bump to pytorch 2.9. There may be additional issues with the iree_boo_experimental backend, as it hasn't been testsed.
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels