Dockerfile

FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
# Install necessary packages
RUN apt update && apt install git build-essential libopenblas-dev wget python3-pip -y
RUN apt update && apt install -y wget
RUN apt-get install -y curl
RUN apt-get install -y nodejs npm
# Set up a new user named "user" with user ID 1000
RUN useradd -m -u 1000 user
# Switch to the "user" user
USER user
# Set home to the user's home directory
ENV HOME=/home/user \
	PATH=/home/user/.local/bin:$PATH

# Set the working directory to the user's home directory
WORKDIR $HOME/app

# Copy the current directory contents into the container at $HOME/app setting the owner to the user
COPY --chown=user . $HOME/app

# Set up the working directory
RUN git clone https://github.com/lolcats399/tabbyAPI/
WORKDIR $HOME/app/tabbyAPI
RUN pip install -r requirements.txt -q
RUN pip install huggingface-hub -q
RUN echo "network:" > config.yml && \
    echo "  host: 0.0.0.0" >> config.yml && \
    echo "  port: 7860" >> config.yml && \
    echo "  disable_auth: False" >> config.yml && \
    echo "" >> config.yml && \
    echo "logging:" >> config.yml && \
    echo "  prompt: False" >> config.yml && \
    echo "  generation_params: False" >> config.yml && \
    echo "" >> config.yml && \
    echo "model:" >> config.yml && \
    echo "  model_dir: models" >> config.yml && \
    echo "  model_name: MiquMaid-v2-2x70B-DPO-exl2-4bpw" >> config.yml && \
    echo "  gpu_split_auto: True" >> config.yml && \
    echo "  no_flash_attention: False" >> config.yml && \
    echo "  cache_mode: FP8" >> config.yml && \
    echo "  num_experts_per_token: 2" >> config.yml && \
    echo "  fasttensors: true" >> config.yml
WORKDIR $HOME/app/tabbyAPI/models
RUN mkdir -p MiquMaid-v2-2x70B-DPO-exl2-4bpw
WORKDIR $HOME/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw
RUN pip install huggingface-hub hf-transfer
ENV HF_HUB_ENABLE_HF_TRANSFER=1
# Ensure the model is downloaded into the current working directory (models folder)
RUN huggingface-cli download alpindale/MiquMaid-v2-2x70B-DPO-exl2-4bpw --local-dir ./ --local-dir-use-symlinks False --cache-dir ~/cache
WORKDIR $HOME/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/config.json?download=true -o $HOME/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/config.json
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/config.json?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/config.json
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/model.safetensors.index.json?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/model.safetensors.index.json
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/special_tokens_map.json?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/special_tokens_map.json
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/tokenizer.json?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/tokenizer.json
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/tokenizer.model?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/tokenizer.model
RUN curl -L https://huggingface.co/NeverSleep/MiquMaid-v2-2x70B/resolve/main/tokenizer_config.json?download=true -o /home/user/app/tabbyAPI/models/MiquMaid-v2-2x70B-DPO-exl2-4bpw/tokenizer_config.json
WORKDIR $HOME/app/tabbyAPI/
EXPOSE 7860
CMD /bin/bash -c "python3 main.py"