test_inference.py

import os

# 1. CUDA runtime DLLs
os.add_dll_directory(
    r"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.8\bin"
)

# 2. Known‑good llama.dll location (from the env where test_cuda.py worked)
# os.add_dll_directory(
#    r"D:\PRATHMESH NIKAM\Downloads\VS\RAGv2\rag\Lib\site-packages\llama_cpp\lib"
# )

from llama_cpp import Llama

MODEL_PATH = "models\gemma-3-4b-it-Q4_0.gguf"

llm = Llama(
    model_path=MODEL_PATH,
    n_gpu_layers=-1,
    n_ctx=4098,
    n_threads=8,
    n_batch=512,
    verbose=True,
)

prompt = "wht is hill climibing algorithm in machine learning?"

# Test with streaming
print("Streaming response:")
stream = llm(prompt, max_tokens=None, temperature=0.7, top_p=0.9, stream=True)
for chunk in stream:
    text = chunk["choices"][0]["text"]
    print(text, end="", flush=True)