Automated deployment of LiteLLM proxy server using Ansible.
This playbook installs Docker, configures LiteLLM proxy, and sets up systemd service management.
Create inventory.ini:
[litellm_servers]
litellm-prod-01 ansible_host=192.168.1.100
litellm-prod-02 ansible_host=192.168.1.101
[litellm_servers:vars]
ansible_user=ubuntu
ansible_python_interpreter=/usr/bin/python3
Create litellm.yml:
---
- name: Deploy LiteLLM Proxy Server
hosts: litellm_servers
become: true
vars:
litellm_version: "main-stable"
litellm_port: 4000
litellm_config_dir: /opt/litellm
litellm_master_key: "{{ vault_litellm_master_key }}"
openai_api_key: "{{ vault_openai_api_key }}"
anthropic_api_key: "{{ vault_anthropic_api_key }}"
tasks:
- name: Install Docker
apt:
name:
- docker.io
- docker-compose
state: present
update_cache: yes
- name: Create LiteLLM directory
file:
path: "{{ litellm_config_dir }}"
state: directory
mode: '0755'
- name: Create LiteLLM config file
copy:
dest: "{{ litellm_config_dir }}/litellm_config.yaml"
content: |
model_list:
# OpenAI
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
# Anthropic
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
general_settings:
master_key: {{ litellm_master_key }}
mode: '0644'
- name: Create environment file
copy:
dest: "{{ litellm_config_dir }}/.env"
content: |
OPENAI_API_KEY={{ openai_api_key }}
ANTHROPIC_API_KEY={{ anthropic_api_key }}
LITELLM_MASTER_KEY={{ litellm_master_key }}
mode: '0600'
- name: Create Docker Compose file
copy:
dest: "{{ litellm_config_dir }}/docker-compose.yml"
content: |
version: "3.9"
services:
litellm:
image: docker.litellm.ai/berriai/litellm:{{ litellm_version }}
ports:
- "{{ litellm_port }}:4000"
volumes:
- ./litellm_config.yaml:/app/config.yaml
env_file:
- .env
command:
- "--config"
- "/app/config.yaml"
- "--port"
- "4000"
- "--num_workers"
- "4"
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:4000/health/liveliness"]
interval: 30s
timeout: 10s
retries: 3
mode: '0644'
- name: Start LiteLLM container
community.docker.docker_compose:
project_src: "{{ litellm_config_dir }}"
state: present
restarted: yes
- name: Wait for LiteLLM to be healthy
uri:
url: "http://localhost:{{ litellm_port }}/health/liveliness"
method: GET
status_code: 200
register: health_check
retries: 10
delay: 5
until: health_check.status == 200
- name: Create systemd service (alternative to Docker)
copy:
dest: /etc/systemd/system/litellm.service
content: |
[Unit]
Description=LiteLLM Proxy Server
After=network.target
[Service]
Type=simple
User=www-data
WorkingDirectory={{ litellm_config_dir }}
Environment="OPENAI_API_KEY={{ openai_api_key }}"
Environment="LITELLM_MASTER_KEY={{ litellm_master_key }}"
ExecStart=/usr/local/bin/litellm --config {{ litellm_config_dir }}/litellm_config.yaml --port 4000
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
notify:
- Restart LiteLLM
when: use_systemd | default(false)
handlers:
- name: Restart LiteLLM
systemd:
name: litellm
state: restarted
daemon_reload: yes
# Basic run
ansible-playbook -i inventory.ini litellm.yml
# With vault secrets
ansible-playbook -i inventory.ini litellm.yml --ask-vault-pass
# Limit to specific host
ansible-playbook -i inventory.ini litellm.yml --limit litellm-prod-01
Create secrets file:
ansible-vault create group_vars/all/vault.yml
Add your secrets:
vault_litellm_master_key: "sk-your-master-key-here"
vault_openai_api_key: "sk-your-openai-key-here"
vault_anthropic_api_key: "sk-ant-your-anthropic-key-here"
# Check service status
ansible -i inventory.ini litellm_servers -m shell -a "docker ps | grep litellm"
# Test health endpoint
ansible -i inventory.ini litellm_servers -m uri -a "url=http://localhost:4000/health/liveliness method=GET"
# View logs
ansible -i inventory.ini litellm_servers -m shell -a "docker logs litellm --tail 20"
For production with multiple providers:
- name: Create multi-provider config
copy:
dest: "{{ litellm_config_dir }}/litellm_config.yaml"
content: |
model_list:
# OpenAI
- model_name: gpt-4o
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
rpm: 100
- model_name: gpt-4o-mini
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
# Anthropic
- model_name: claude-sonnet
litellm_params:
model: anthropic/claude-sonnet-4-20250514
api_key: os.environ/ANTHROPIC_API_KEY
rpm: 60
# Azure OpenAI
- model_name: azure-gpt-4o
litellm_params:
model: azure/gpt-4o
api_base: {{ azure_api_base }}
api_key: os.environ/AZURE_API_KEY
api_version: "2025-01-01-preview"
# Redis for caching (if using)
router_settings:
redis_host: {{ redis_host | default(omit) }}
redis_port: 6379
# Enable caching
litellm_settings:
cache: true
general_settings:
master_key: {{ litellm_master_key }}
- name: Rolling update LiteLLM
hosts: litellm_servers
become: true
serial: 1 # Update one server at a time
tasks:
- name: Pull latest image
community.docker.docker_image:
name: docker.litellm.ai/berriai/litellm:main-stable
source: pull
- name: Recreate container
community.docker.docker_compose:
project_src: /opt/litellm
state: present
restarted: yes
remove_orphans: yes
- name: Wait for healthy
uri:
url: "http://localhost:4000/health/liveliness"
method: GET
register: result
retries: 10
delay: 5
until: result.status == 200
- name: Backup LiteLLM configuration
hosts: litellm_servers
become: true
tasks:
- name: Create backup directory
file:
path: /opt/litellm/backups
state: directory
- name: Backup config file
archive:
path: /opt/litellm/litellm_config.yaml
dest: "/opt/litellm/backups/config-{{ ansible_date_time.date }}.tar.gz"
ansible -i inventory.ini litellm_servers \
-m shell -a "docker ps -a | grep litellm"
ansible -i inventory.ini litellm_servers \
-m shell -a "docker logs litellm --tail 50"
ansible -i inventory.ini litellm_servers \
-m shell -a "docker compose -f /opt/litellm/docker-compose.yml restart"
See the main LiteLLM Setup guide for more details.