feat(first-commit): follow examples and tutorials

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+.idea

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.12
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,12 +1,14 @@
 ---
-title: Vllm Inference
-emoji: 🚀
+title: Deploy VLLM
+emoji: 🐢
 colorFrom: blue
-colorTo: green
+colorTo: blue
 sdk: docker
 pinned: false
-license: apache-2.0
-short_description: VLLM Inference Engine Demo
 ---
-
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+```shell
+poetry export -f requirements.txt --output requirements.txt --without-hashes
+```

main.py

@@ -0,0 +1,61 @@
+from typing import Optional
+from fastapi import FastAPI
+from pydantic import BaseModel
+from vllm import LLM, SamplingParams, RequestOutput
+
+
+# Don't forget to set HF_TOKEN in the env during running
+
+app = FastAPI()
+
+# Initialize the LLM engine
+# Replace 'your-model-path' with the actual path or name of your model
+
+engine = LLM(
+    model='meta-llama/Llama-3.2-3B-Instruct',
+    revision="0cb88a4f764b7a12671c53f0838cd831a0843b95",
+    max_num_batched_tokens=512,    # Reduced for T4
+    max_num_seqs=16,               # Reduced for T4
+    gpu_memory_utilization=0.85,   # Slightly increased, adjust if needed
+    max_model_len=131072,          # Llama-3.2-3B-Instruct context length
+    enforce_eager=True,            # Disable CUDA graph
+    dtype='half',                  # Use half precision
+)
+
+
+@app.get("/")
+def greet_json():
+    return {"Hello": "World!"}
+
+
+class GenerationRequest(BaseModel):
+    prompt: str
+    max_tokens: int = 100
+    temperature: float = 0.7
+    logit_bias: Optional[dict[int, float]] = None
+
+
+class GenerationResponse(BaseModel):
+    text: Optional[str]
+    error: Optional[str]
+
+
+@app.post("/generate-llama3-2")
+def generate_text(request: GenerationRequest) -> list[RequestOutput] | dict[str, str]:
+    try:
+        sampling_params: SamplingParams = SamplingParams(
+            temperature=request.temperature,
+            max_tokens=request.max_tokens,
+            logit_bias=request.logit_bias,
+        )
+
+        # Generate text
+        return engine.generate(
+            prompts=request.prompt,
+            sampling_params=sampling_params
+        )
+
+    except Exception as e:
+        return {
+            "error": str(e)
+        }

pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+package-mode = false
+name = "gradio-qa"
+version = "0.1.0"
+description = ""
+authors = ["Yusuf <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.12,<3.13"
+vllm = "^0.6.4.post1"
+fastapi = "^0.115.5"
+pydantic = "^2.10.2"
+uvicorn = "^0.32.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,119 @@
+aiohappyeyeballs==2.4.3 ; python_version >= "3.12" and python_version < "3.13"
+aiohttp==3.11.7 ; python_version >= "3.12" and python_version < "3.13"
+aiosignal==1.3.1 ; python_version >= "3.12" and python_version < "3.13"
+annotated-types==0.7.0 ; python_version >= "3.12" and python_version < "3.13"
+anyio==4.6.2.post1 ; python_version >= "3.12" and python_version < "3.13"
+attrs==24.2.0 ; python_version >= "3.12" and python_version < "3.13"
+certifi==2024.8.30 ; python_version >= "3.12" and python_version < "3.13"
+cffi==1.17.1 ; python_version >= "3.12" and python_version < "3.13" and implementation_name == "pypy"
+charset-normalizer==3.4.0 ; python_version >= "3.12" and python_version < "3.13"
+click==8.1.7 ; python_version >= "3.12" and python_version < "3.13"
+cloudpickle==3.1.0 ; python_version >= "3.12" and python_version < "3.13"
+colorama==0.4.6 ; python_version >= "3.12" and python_version < "3.13" and (platform_system == "Windows" or sys_platform == "win32")
+compressed-tensors==0.8.0 ; python_version >= "3.12" and python_version < "3.13"
+datasets==2.14.4 ; python_version >= "3.12" and python_version < "3.13"
+dill==0.3.7 ; python_version >= "3.12" and python_version < "3.13"
+diskcache==5.6.3 ; python_version >= "3.12" and python_version < "3.13"
+distro==1.9.0 ; python_version >= "3.12" and python_version < "3.13"
+einops==0.8.0 ; python_version >= "3.12" and python_version < "3.13"
+fastapi==0.115.5 ; python_version >= "3.12" and python_version < "3.13"
+filelock==3.16.1 ; python_version >= "3.12" and python_version < "3.13"
+frozenlist==1.5.0 ; python_version >= "3.12" and python_version < "3.13"
+fsspec==2024.10.0 ; python_version >= "3.12" and python_version < "3.13"
+fsspec[http]==2024.10.0 ; python_version >= "3.12" and python_version < "3.13"
+gguf==0.10.0 ; python_version >= "3.12" and python_version < "3.13"
+h11==0.14.0 ; python_version >= "3.12" and python_version < "3.13"
+httpcore==1.0.7 ; python_version >= "3.12" and python_version < "3.13"
+httptools==0.6.4 ; python_version >= "3.12" and python_version < "3.13"
+httpx==0.27.2 ; python_version >= "3.12" and python_version < "3.13"
+huggingface-hub==0.26.2 ; python_version >= "3.12" and python_version < "3.13"
+idna==3.10 ; python_version >= "3.12" and python_version < "3.13"
+importlib-metadata==8.5.0 ; python_version >= "3.12" and python_version < "3.13"
+interegular==0.3.3 ; python_version >= "3.12" and python_version < "3.13"
+jinja2==3.1.4 ; python_version >= "3.12" and python_version < "3.13"
+jiter==0.8.0 ; python_version >= "3.12" and python_version < "3.13"
+jsonschema-specifications==2024.10.1 ; python_version >= "3.12" and python_version < "3.13"
+jsonschema==4.23.0 ; python_version >= "3.12" and python_version < "3.13"
+lark==1.2.2 ; python_version >= "3.12" and python_version < "3.13"
+llvmlite==0.43.0 ; python_version >= "3.12" and python_version < "3.13"
+lm-format-enforcer==0.10.9 ; python_version >= "3.12" and python_version < "3.13"
+markupsafe==3.0.2 ; python_version >= "3.12" and python_version < "3.13"
+mistral-common[opencv]==1.5.1 ; python_version >= "3.12" and python_version < "3.13"
+mpmath==1.3.0 ; python_version >= "3.12" and python_version < "3.13"
+msgpack==1.1.0 ; python_version >= "3.12" and python_version < "3.13"
+msgspec==0.18.6 ; python_version >= "3.12" and python_version < "3.13"
+multidict==6.1.0 ; python_version >= "3.12" and python_version < "3.13"
+multiprocess==0.70.15 ; python_version >= "3.12" and python_version < "3.13"
+nest-asyncio==1.6.0 ; python_version >= "3.12" and python_version < "3.13"
+networkx==3.4.2 ; python_version >= "3.12" and python_version < "3.13"
+numba==0.60.0 ; python_version >= "3.12" and python_version < "3.13"
+numpy==1.26.4 ; python_version >= "3.12" and python_version < "3.13"
+nvidia-cublas-cu12==12.4.5.8 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cuda-cupti-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cuda-nvrtc-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cuda-runtime-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cudnn-cu12==9.1.0.70 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cufft-cu12==11.2.1.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-curand-cu12==10.3.5.147 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cusolver-cu12==11.6.1.9 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-cusparse-cu12==12.3.1.170 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-ml-py==12.560.30 ; python_version >= "3.12" and python_version < "3.13"
+nvidia-nccl-cu12==2.21.5 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-nvjitlink-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+nvidia-nvtx-cu12==12.4.127 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+openai==1.55.1 ; python_version >= "3.12" and python_version < "3.13"
+opencv-python-headless==4.10.0.84 ; python_version >= "3.12" and python_version < "3.13"
+outlines==0.0.46 ; python_version >= "3.12" and python_version < "3.13"
+packaging==24.2 ; python_version >= "3.12" and python_version < "3.13"
+pandas==2.2.3 ; python_version >= "3.12" and python_version < "3.13"
+partial-json-parser==0.2.1.1.post4 ; python_version >= "3.12" and python_version < "3.13"
+pillow==10.4.0 ; python_version >= "3.12" and python_version < "3.13"
+prometheus-client==0.21.0 ; python_version >= "3.12" and python_version < "3.13"
+prometheus-fastapi-instrumentator==7.0.0 ; python_version >= "3.12" and python_version < "3.13"
+propcache==0.2.0 ; python_version >= "3.12" and python_version < "3.13"
+protobuf==5.28.3 ; python_version >= "3.12" and python_version < "3.13"
+psutil==6.1.0 ; python_version >= "3.12" and python_version < "3.13"
+py-cpuinfo==9.0.0 ; python_version >= "3.12" and python_version < "3.13"
+pyairports==2.1.1 ; python_version >= "3.12" and python_version < "3.13"
+pyarrow==18.1.0 ; python_version >= "3.12" and python_version < "3.13"
+pycountry==24.6.1 ; python_version >= "3.12" and python_version < "3.13"
+pycparser==2.22 ; python_version >= "3.12" and python_version < "3.13" and implementation_name == "pypy"
+pydantic-core==2.27.1 ; python_version >= "3.12" and python_version < "3.13"
+pydantic==2.10.2 ; python_version >= "3.12" and python_version < "3.13"
+python-dateutil==2.9.0.post0 ; python_version >= "3.12" and python_version < "3.13"
+python-dotenv==1.0.1 ; python_version >= "3.12" and python_version < "3.13"
+pytz==2024.2 ; python_version >= "3.12" and python_version < "3.13"
+pyyaml==6.0.2 ; python_version >= "3.12" and python_version < "3.13"
+pyzmq==26.2.0 ; python_version >= "3.12" and python_version < "3.13"
+ray==2.39.0 ; python_version >= "3.12" and python_version < "3.13"
+referencing==0.35.1 ; python_version >= "3.12" and python_version < "3.13"
+regex==2024.11.6 ; python_version >= "3.12" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.12" and python_version < "3.13"
+rpds-py==0.21.0 ; python_version >= "3.12" and python_version < "3.13"
+safetensors==0.4.5 ; python_version >= "3.12" and python_version < "3.13"
+sentencepiece==0.2.0 ; python_version >= "3.12" and python_version < "3.13"
+setuptools==75.6.0 ; python_version >= "3.12" and python_version < "3.13"
+six==1.16.0 ; python_version >= "3.12" and python_version < "3.13"
+sniffio==1.3.1 ; python_version >= "3.12" and python_version < "3.13"
+starlette==0.41.3 ; python_version >= "3.12" and python_version < "3.13"
+sympy==1.13.1 ; python_version >= "3.12" and python_version < "3.13"
+tiktoken==0.7.0 ; python_version >= "3.12" and python_version < "3.13"
+tokenizers==0.20.4 ; python_version >= "3.12" and python_version < "3.13"
+torch==2.5.1 ; python_version >= "3.12" and python_version < "3.13"
+torchvision==0.20.1 ; python_version >= "3.12" and python_version < "3.13"
+tqdm==4.67.1 ; python_version >= "3.12" and python_version < "3.13"
+transformers==4.46.3 ; python_version >= "3.12" and python_version < "3.13"
+triton==3.1.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version < "3.13" and python_version >= "3.12"
+typing-extensions==4.12.2 ; python_version >= "3.12" and python_version < "3.13"
+tzdata==2024.2 ; python_version >= "3.12" and python_version < "3.13"
+urllib3==2.2.3 ; python_version >= "3.12" and python_version < "3.13"
+uvicorn==0.32.1 ; python_version >= "3.12" and python_version < "3.13"
+uvicorn[standard]==0.32.1 ; python_version >= "3.12" and python_version < "3.13"
+uvloop==0.21.0 ; (sys_platform != "win32" and sys_platform != "cygwin") and platform_python_implementation != "PyPy" and python_version >= "3.12" and python_version < "3.13"
+vllm==0.6.4.post1 ; python_version >= "3.12" and python_version < "3.13"
+watchfiles==1.0.0 ; python_version >= "3.12" and python_version < "3.13"
+websockets==14.1 ; python_version >= "3.12" and python_version < "3.13"
+xformers==0.0.28.post3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.12" and python_version < "3.13"
+xxhash==3.5.0 ; python_version >= "3.12" and python_version < "3.13"
+yarl==1.18.0 ; python_version >= "3.12" and python_version < "3.13"
+zipp==3.21.0 ; python_version >= "3.12" and python_version < "3.13"