Upload folder using huggingface_hub

README.md

@@ -1,5 +1,6 @@
 ---
 thumbnail: "https://assets-global.website-files.com/646b351987a8d8ce158d1940/64ec9e96b4334c0e1ac41504_Logo%20with%20white%20text.svg"
+base_model: MaziyarPanahi/Llama-3-8B-Instruct-64k
 metrics:
 - memory_disk
 - memory_inference
@@ -59,9 +60,9 @@ You can run the smashed model with these steps:
 2. Load & run the model.
     ```python 
    from transformers import AutoModelForCausalLM, AutoTokenizer
-   
+   from awq import AutoAWQForCausalLM
 
-   model = AutoModelForCausalLM.from_pretrained("PrunaAI/MaziyarPanahi-Llama-3-8B-Instruct-64k-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
+   model = AutoAWQForCausalLM.from_quantized("PrunaAI/MaziyarPanahi-Llama-3-8B-Instruct-64k-AWQ-4bit-smashed", trust_remote_code=True, device_map='auto')
    tokenizer = AutoTokenizer.from_pretrained("MaziyarPanahi/Llama-3-8B-Instruct-64k")
     
    input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmptiupqo16",
+  "_name_or_path": "/tmp/tmpe413c_m0",
   "architectures": [
     "LlamaForCausalLM"
   ],

results.json

@@ -1,26 +1,30 @@
 {
     "base_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "base_current_gpu_total_memory": 40339.3125,
-    "base_token_generation_latency_sync": 101.87378540039063,
-    "base_token_generation_latency_async": 101.67951695621014,
-    "base_token_generation_throughput_sync": 0.009816067951826256,
-    "base_token_generation_throughput_async": 0.009834822488689296,
-    "base_token_generation_CO2_emissions": 2.6819935676011093e-05,
-    "base_token_generation_energy_consumption": 0.008565413871084828,
-    "base_inference_latency_sync": 90.84702758789062,
-    "base_inference_latency_async": 88.71569633483887,
-    "base_inference_throughput_sync": 0.011007514792187808,
-    "base_inference_throughput_async": 0.01127196247466411,
+    "base_token_generation_latency_sync": 54.22916030883789,
+    "base_token_generation_latency_async": 54.13356442004442,
+    "base_token_generation_throughput_sync": 0.018440263398971105,
+    "base_token_generation_throughput_async": 0.018472827546336907,
+    "base_token_generation_CO2_emissions": null,
+    "base_token_generation_energy_consumption": null,
+    "base_inference_latency_sync": 52.29578285217285,
+    "base_inference_latency_async": 51.306843757629395,
+    "base_inference_throughput_sync": 0.019122000770630986,
+    "base_inference_throughput_async": 0.019490577216636887,
+    "base_inference_CO2_emissions": null,
+    "base_inference_energy_consumption": null,
     "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB",
     "smashed_current_gpu_total_memory": 40339.3125,
-    "smashed_token_generation_latency_sync": 63.452283477783205,
-    "smashed_token_generation_latency_async": 64.33862000703812,
-    "smashed_token_generation_throughput_sync": 0.015759874116274063,
-    "smashed_token_generation_throughput_async": 0.01554276420430852,
-    "smashed_token_generation_CO2_emissions": 1.9680748845555618e-05,
-    "smashed_token_generation_energy_consumption": 0.0053972414185424426,
-    "smashed_inference_latency_sync": 76.47395782470703,
-    "smashed_inference_latency_async": 73.17061424255371,
-    "smashed_inference_throughput_sync": 0.013076346882584418,
-    "smashed_inference_throughput_async": 0.013666688606509356
+    "smashed_token_generation_latency_sync": 41.130767822265625,
+    "smashed_token_generation_latency_async": 40.58046396821737,
+    "smashed_token_generation_throughput_sync": 0.024312699542133578,
+    "smashed_token_generation_throughput_async": 0.024642399376783867,
+    "smashed_token_generation_CO2_emissions": null,
+    "smashed_token_generation_energy_consumption": null,
+    "smashed_inference_latency_sync": 51.22703399658203,
+    "smashed_inference_latency_async": 39.65771198272705,
+    "smashed_inference_throughput_sync": 0.01952094279100215,
+    "smashed_inference_throughput_async": 0.025215776453153697,
+    "smashed_inference_CO2_emissions": null,
+    "smashed_inference_energy_consumption": null
 }

smash_config.json

@@ -2,19 +2,19 @@
     "api_key": null,
     "verify_url": "http://johnrachwan.pythonanywhere.com",
     "smash_config": {
-        "pruners": "[]",
+        "pruners": "None",
         "pruning_ratio": 0.0,
-        "factorizers": "[]",
+        "factorizers": "None",
         "quantizers": "['awq']",
         "weight_quantization_bits": 4,
-        "output_deviation": 0.01,
-        "compilers": "[]",
+        "output_deviation": 0.005,
+        "compilers": "None",
         "static_batch": true,
         "static_shape": true,
         "controlnet": "None",
         "unet_dim": 4,
         "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/models78wd5963",
+        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsvigq773l",
         "batch_size": 1,
         "model_name": "MaziyarPanahi/Llama-3-8B-Instruct-64k",
         "task": "text_text_generation",