version: "3.9"
services:
  # mt:
  #   image: cerebrumtech/mt:apr5
  #   restart: on-failure
  #   runtime: nvidia
  #   healthcheck:
  #     test: [ "CMD-SHELL", "curl --location 'http://localhost:8000/api/v1/translate' --header 'Content-Type: application/json' --data '{\"src\":\"en\", \"tgt\":\"tr\", \"text\":\"hi\"}' || exit 1" ]
  #     interval: 120s
  #     timeout: 20s
  #     retries: 2
  #     start_period: 90s
  #   ports:
  #     - 8009:8000
  #   volumes:
  #     - ./mt/models:/app/models
  #     - ./mt/config.json:/app/config.json
  #   environment:
  #     - MT_API_CONFIG=/app/config.json
  #     - MT_API_DEVICE=gpu #or gpu, if so make runtime:nvidia
  #     - MT_API_THREADS=6
  #     - MODELS_ROOT=/app/models
  #     #     - NVIDIA_VISIBLE_DEVICES=2
  #     - NVIDIA_DRIVER_CAPABILITIES=all
  #     - VIRTUAL_HOST=mt.${DOMAIN}
  #     - CERT_NAME=${DOMAIN}
  #     - max_length=4000
  #     - CUDA_LAUNCH_BLOCKING=1
  #   dns: 0.0.0.0

  # transcirption:
  #   image: cerebrumtech/transcribe
  #   restart: on-failure
  #   ports:
  #     - 8011:5000
  #   environment:
  #     - VIRTUAL_HOST=transcribe.${DOMAIN}
  #     - CERT_NAME=${DOMAIN}
  #   dns: 0.0.0.0

  # text-embeddings-inference:
  #   image: ghcr.io/huggingface/text-embeddings-inference:1.6
  #   command: --model-id intfloat/multilingual-e5-large-instruct
  #   runtime: nvidia
  #   restart: unless-stopped
  #   environment:
  #     - VIRTUAL_HOST=embeddings.${DOMAIN}
  #     - CERT_NAME=${DOMAIN}
  #     - HUGGINGFACE_OFFLINE=1
  #     # - NVIDIA_VISIBLE_DEVICES=2
  #   volumes:
  #     - ./embedding/data:/data
  #   ports:
  #     - "8092:80"
  #   dns: 0.0.0.0
  llmapi:
    image: vllm/vllm-openai:latest
    container_name: llmapi
    restart: on-failure
    shm_size: 15.91gb
    command:
      [
        "--model",
        "unsloth/Llama-3.3-70B-Instruct-bnb-4bit",
        "--served-model-name",
        "llama3-8b",
        "another-model",
        "--gpu-memory-utilization",
        "0.85",
        "--swap-space",
        "1",
        "--enforce-eager",
      ]
    # healthcheck:
    #   test:
    #     [
    #       "CMD-SHELL",
    #       "curl --location 'http://localhost:8000/v1/chat/completions' --header 'Content-Type: application/json' --data '{\"model\": \"/root/models/Mixtral-8x7B-Instruct-v0.1-GPTQ\", \"messages\": [{\"role\": \"user\", \"content\": \"say just hi\"}], \"temperature\": 0, \"top_p\": 1, \"max_tokens\": 50, \"presence_penalty\": 0.3, \"stream\": false, \"safe_prompt\": false, \"frequency_penalty\": 0.6, \"random_seed\": null}' || exit 1"
    #     ]
    #   interval: 150s
    #   timeout: 20s
    #   retries: 2
    #   start_period: 90s
    environment:
      - VIRTUAL_HOST=llm.${DOMAIN}
      - CERT_NAME=${DOMAIN}
      - HUGGING_FACE_HUB_TOKEN="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"
      # - TRANSFORMERS_OFFLINE=1
      # - HF_HUB_OFFLINE=1
    volumes:
      - ./llm/cache/huggingface:/root/.cache/huggingface
      - ./llm/models:/root/models
    ports:
      - "8008:8000"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    # dns: 0.0.0.0
    # summarizer:
    #   image: cerebrumtech/summarizer:latest
    #   runtime: nvidia
    #   restart: unless-stopped
    #   environment:
    #     - VIRTUAL_HOST=summarizer.${DOMAIN}
    #     - CERT_NAME=${DOMAIN}
    #     - NVIDIA_VISIBLE_DEVICES=2
    #   volumes:
    #     - ./llm/summarizer/cache:/root
    #   ports:
    #     - "8010:8080"
    #   dns: 0.0.0.0
    # tts:
    #   image: cerebrumtech/tts-api:latest
    #   restart: unless-stopped
    #   runtime: nvidia
    #   ports:
    #     - 8012:5000
    #   environment:
    #     - VIRTUAL_HOST=tts.${DOMAIN}
    #     - CERT_NAME=${DOMAIN}
    #     - NVIDIA_VISIBLE_DEVICES=2
    #   dns: 0.0.0.0
networks:
  default:
    name: proxy
    external: true
  backend:
