Update README.md

README.md

@@ -72,3 +72,62 @@ if(backend == 'gemlite'):
     gemlite.core.GemLiteLinear.cache_config('/tmp/gemlite_config.json') 
 ```
 
+Run with <a href="https://github.com/vllm-project/vllm/">vllm</a>:
+```Python
+##################################################################
+import torch
+import torch.nn as nn
+from typing import Optional
+from vllm.model_executor.layers.linear import RowParallelLinear
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
+class MixtralMLPRowParallel(nn.Module):
+
+    def __init__(
+        self,
+        num_experts: int,
+        hidden_size: int,
+        intermediate_size: int,
+        quant_config: Optional[QuantizationConfig] = None,
+    ) -> None:
+        super().__init__()
+        self.num_experts = num_experts
+        self.ffn_dim = intermediate_size
+        self.hidden_dim = hidden_size
+
+        self.w1 = RowParallelLinear(self.hidden_dim,
+                                   self.ffn_dim,
+                                   bias=False,
+                                   quant_config=quant_config)
+        self.w2 = RowParallelLinear(self.ffn_dim,
+                                   self.hidden_dim,
+                                   bias=False,
+                                   quant_config=quant_config)
+        self.w3 = RowParallelLinear(self.hidden_dim,
+                                   self.ffn_dim,
+                                   bias=False,
+                                   quant_config=quant_config)
+
+        # TODO: Use vllm's SiluAndMul
+        self.act_fn = nn.SiLU()
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        w1_out, _ = self.w1(hidden_states)
+        w1_out = self.act_fn(w1_out)
+        w3_out, _ = self.w3(hidden_states)
+        current_hidden_states = w1_out * w3_out
+        current_hidden_states, _ = self.w2(current_hidden_states)
+        return current_hidden_states
+
+import vllm.model_executor.models.mixtral_quant as mixtral_quant
+mixtral_quant.MixtralMLP = MixtralMLPRowParallel
+##################################################################
+
+from vllm import LLM
+from vllm.sampling_params import SamplingParams
+model_id = "mobiuslabsgmbh/Mixtral-8x7B-Instruct-v0.1_4bitgs64_hqq_hf"
+
+llm = LLM(model=model_id, gpu_memory_utilization=0.80)
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=1024)
+outputs = llm.generate(["What is the capital of Germany?"], sampling_params)
+print(outputs[0].outputs[0].text)
+```