From 71bb1f5e17c6b4947adb5c0c9f9403fbaa1cc030 Mon Sep 17 00:00:00 2001 From: Alex Renoki Date: Sat, 17 Aug 2024 06:20:42 +0300 Subject: [PATCH] Added litellm --- public/svgs/litellm.svg | 1 + templates/compose/litellm.yaml | 177 +++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 public/svgs/litellm.svg create mode 100644 templates/compose/litellm.yaml diff --git a/public/svgs/litellm.svg b/public/svgs/litellm.svg new file mode 100644 index 000000000..01830c3f6 --- /dev/null +++ b/public/svgs/litellm.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/compose/litellm.yaml b/templates/compose/litellm.yaml new file mode 100644 index 000000000..b16e08fbe --- /dev/null +++ b/templates/compose/litellm.yaml @@ -0,0 +1,177 @@ +# documentation: https://docs.litellm.ai +# slogan: Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate, Groq (100+ LLMs) +# tags: ai, qdrant, weaviate, langchain, openai, gpt, llm, lmops, anthropic, cohere, ollama, sagemaker, huggingface, replicate, groq +# logo: svgs/litellm.svg +# port: 4000 + +volumes: + pg-data: null + redis-data: null + +services: + litellm: + image: "ghcr.io/berriai/litellm-database:main-latest" + restart: always + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + environment: + - LITELLM_LOG=ERROR + - LITELLM_MODE=PRODUCTION + - "LITELLM_MASTER_KEY=${SERVICE_PASSWORD_32_LITELLM_MASTER_KEY}" + - "UI_USERNAME=${SERVICE_USER_UI}" + - "UI_PASSWORD=${SERVICE_PASSWORD_UI}" + - "DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@postgres:5432/${POSTGRES_DB:-litellm}" + - REDIS_HOST=redis + - REDIS_PORT=6379 + - "POSTGRES_USER=${SERVICE_USER_POSTGRES}" + - "POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES}" + - "POSTGRES_DB=${POSTGRES_DB:-litellm}" + - "OPENAI_API_KEY=${OPENAI_API_KEY}" + - "OPENAI_API_BASE=${OPENAI_API_BASE}" + - "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" + - "ANTHROPIC_API_BASE=${ANTHROPIC_API_BASE}" + - "VOYAGE_API_KEY=${VOYAGE_API_KEY}" + - "VOYAGE_API_BASE=${VOYAGE_API_BASE}" + volumes: + - "./litellm-config.yaml:/app/config.yaml" + - type: bind + source: ./docker/app/config.yaml + target: /app/config.yaml + content: | + general_settings: + proxy_batch_write_at: 60 + + router_settings: + routing_strategy: usage-based-routing-v2 + redis_host: os.environ/REDIS_HOST + redis_port: os.environ/REDIS_PORT + redis_password: os.environ/REDIS_PASSWORD + enable_pre_call_check: true + + litellm_settings: + set_verbose: false + json_logs: true + log_raw_request_response: true + # turn_off_message_logging: false + # redact_user_api_key_info: false + drop_params: true + # max_budget: 100 + # budget_duration: 30d + num_retries: 3 + request_timeout: 600 + telemetry: false + cache: true + cache_params: + type: redis + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD + namespace: "litellm_cache" + ttl: 600 + # success_callback: + # - "langfuse" + # - "prometheus" + # failure_callback: + # - "langfuse" + # - "prometheus" + model_list: + # OpenAI + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + api_base: os.environ/OPENAI_API_BASE + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + api_base: os.environ/OPENAI_API_BASE + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + api_base: os.environ/OPENAI_API_BASE + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + api_base: os.environ/OPENAI_API_BASE + # Anthropic + - model_name: claude-3-haiku + litellm_params: + model: claude-3-haiku-20240307 + api_key: "os.environ/ANTHROPIC_API_KEY" + api_base: "os.environ/ANTHROPIC_API_BASE" + - model_name: claude-3.5-sonnet + litellm_params: + model: claude-3-5-sonnet-20240620 + api_key: "os.environ/ANTHROPIC_API_KEY" + api_base: "os.environ/ANTHROPIC_API_BASE" + # VoyageAI + - model_name: voyage-law-2 + model_info: + output_vector_size: 1024 + litellm_params: + model: voyage/voyage-law-2 + api_key: "os.environ/VOYAGE_API_KEY" + api_base: "os.environ/VOYAGE_API_BASE" + rpm: 300 + tpm: 1000000 + - model_name: voyage-multilingual-2 + model_info: + mode: embedding + max_tokens: 32000 + max_input_tokens: 32000 + output_vector_size: 1024 + litellm_params: + model: voyage/voyage-multilingual-2 + api_key: "os.environ/VOYAGE_API_KEY" + api_base: "os.environ/VOYAGE_API_BASE" + input_cost_per_token: 0.00000012 + output_cost_per_token: 0 + rpm: 300 + tpm: 1000000 + healthcheck: + test: + - CMD + - python + - "-c" + - "import requests as r;r.get('http://127.0.0.1:4000/health/liveliness').raise_for_status()" + interval: 5s + timeout: 5s + retries: 3 + command: + - "--config" + - /app/config.yaml + - "--port" + - "4000" + - "--num_workers" + - "8" + postgres: + image: "postgres:16-alpine" + environment: + - POSTGRES_DB=${POSTGRES_DB:-litellm} + - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - POSTGRES_USER=$SERVICE_USER_POSTGRES + volumes: + - "pg-data:/var/lib/postgresql/data" + healthcheck: + test: + - CMD-SHELL + - "pg_isready -h localhost -U ${POSTGRES_USER} -d ${POSTGRES_DB}" + interval: 5s + timeout: 5s + retries: 3 + redis: + image: redis:7-alpine + command: redis-server --appendonly yes + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 3