Added litellm

public/svgs/litellm.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 36 36"><path fill="#939598" d="M0 34h36v2H0z"/><path fill="#D1D3D4" d="M3 35h33V6H21c-4 0-5 2-5 2l-4 6S0 19 0 24c0 4 6 6 6 6z"/><path fill="#231F20" d="m14 35 2-3h20v3z"/><path fill="#3B88C3" d="M0 23.999c0 4 6 6 6 6V17.125C3 19 0 21.499 0 23.999M6 30v-.001z"/><path fill="#269" d="m6 30-3 5h33v-5z"/><path fill="#3B88C3" d="m20 30 4-6h12v6z"/><path fill="#55ACEE" d="M26 8H16l-4 6h-.001 10.843c.477 0 1.108-.448 1.412-1l2.197-4c.303-.552.102-1-.451-1"/><path fill="#3B88C3" d="m25.902 10 .549-1c.303-.552.102-1-.451-1H16l-1.333 2z"/></svg>

templates/compose/litellm.yaml

@@ -0,0 +1,177 @@
+# documentation: https://docs.litellm.ai
+# slogan: Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate, Groq (100+ LLMs)
+# tags: ai, qdrant, weaviate, langchain, openai, gpt, llm, lmops, anthropic, cohere, ollama, sagemaker, huggingface, replicate, groq
+# logo: svgs/litellm.svg
+# port: 4000
+
+volumes:
+  pg-data: null
+  redis-data: null
+
+services:
+  litellm:
+    image: "ghcr.io/berriai/litellm-database:main-latest"
+    restart: always
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      - LITELLM_LOG=ERROR
+      - LITELLM_MODE=PRODUCTION
+      - "LITELLM_MASTER_KEY=${SERVICE_PASSWORD_32_LITELLM_MASTER_KEY}"
+      - "UI_USERNAME=${SERVICE_USER_UI}"
+      - "UI_PASSWORD=${SERVICE_PASSWORD_UI}"
+      - "DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@postgres:5432/${POSTGRES_DB:-litellm}"
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - "POSTGRES_USER=${SERVICE_USER_POSTGRES}"
+      - "POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES}"
+      - "POSTGRES_DB=${POSTGRES_DB:-litellm}"
+      - "OPENAI_API_KEY=${OPENAI_API_KEY}"
+      - "OPENAI_API_BASE=${OPENAI_API_BASE}"
+      - "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}"
+      - "ANTHROPIC_API_BASE=${ANTHROPIC_API_BASE}"
+      - "VOYAGE_API_KEY=${VOYAGE_API_KEY}"
+      - "VOYAGE_API_BASE=${VOYAGE_API_BASE}"
+    volumes:
+      - "./litellm-config.yaml:/app/config.yaml"
+      - type: bind
+        source: ./docker/app/config.yaml
+        target: /app/config.yaml
+        content: |
+          general_settings:
+            proxy_batch_write_at: 60
+
+          router_settings:
+            routing_strategy: usage-based-routing-v2
+            redis_host: os.environ/REDIS_HOST
+            redis_port: os.environ/REDIS_PORT
+            redis_password: os.environ/REDIS_PASSWORD
+            enable_pre_call_check: true
+
+          litellm_settings:
+            set_verbose: false
+            json_logs: true
+            log_raw_request_response: true
+            # turn_off_message_logging: false
+            # redact_user_api_key_info: false
+            drop_params: true
+            # max_budget: 100
+            # budget_duration: 30d
+            num_retries: 3
+            request_timeout: 600
+            telemetry: false
+            cache: true
+            cache_params:
+              type: redis
+              host: os.environ/REDIS_HOST
+              port: os.environ/REDIS_PORT
+              password: os.environ/REDIS_PASSWORD
+              namespace: "litellm_cache"
+              ttl: 600
+            # success_callback:
+            #   - "langfuse"
+            #   - "prometheus"
+            # failure_callback:
+            #   - "langfuse"
+            #   - "prometheus"
+          model_list:
+            # OpenAI
+            - model_name: gpt-3.5-turbo
+              litellm_params:
+                model: openai/gpt-3.5-turbo
+                api_key: os.environ/OPENAI_API_KEY
+                api_base: os.environ/OPENAI_API_BASE
+            - model_name: gpt-4
+              litellm_params:
+                model: openai/gpt-4
+                api_key: os.environ/OPENAI_API_KEY
+                api_base: os.environ/OPENAI_API_BASE
+            - model_name: gpt-4o
+              litellm_params:
+                model: openai/gpt-4o
+                api_key: os.environ/OPENAI_API_KEY
+                api_base: os.environ/OPENAI_API_BASE
+            - model_name: gpt-4o-mini
+              litellm_params:
+                model: openai/gpt-4o-mini
+                api_key: os.environ/OPENAI_API_KEY
+                api_base: os.environ/OPENAI_API_BASE
+            # Anthropic
+            - model_name: claude-3-haiku
+              litellm_params:
+                model: claude-3-haiku-20240307
+                api_key: "os.environ/ANTHROPIC_API_KEY"
+                api_base: "os.environ/ANTHROPIC_API_BASE"
+            - model_name: claude-3.5-sonnet
+              litellm_params:
+                model: claude-3-5-sonnet-20240620
+                api_key: "os.environ/ANTHROPIC_API_KEY"
+                api_base: "os.environ/ANTHROPIC_API_BASE"
+            # VoyageAI
+            - model_name: voyage-law-2
+              model_info:
+                output_vector_size: 1024
+              litellm_params:
+                model: voyage/voyage-law-2
+                api_key: "os.environ/VOYAGE_API_KEY"
+                api_base: "os.environ/VOYAGE_API_BASE"
+                rpm: 300
+                tpm: 1000000
+            - model_name: voyage-multilingual-2
+              model_info:
+                mode: embedding
+                max_tokens: 32000
+                max_input_tokens: 32000
+                output_vector_size: 1024
+              litellm_params:
+                model: voyage/voyage-multilingual-2
+                api_key: "os.environ/VOYAGE_API_KEY"
+                api_base: "os.environ/VOYAGE_API_BASE"
+                input_cost_per_token: 0.00000012
+                output_cost_per_token: 0
+                rpm: 300
+                tpm: 1000000
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - "-c"
+        - "import requests as r;r.get('http://127.0.0.1:4000/health/liveliness').raise_for_status()"
+      interval: 5s
+      timeout: 5s
+      retries: 3
+    command:
+      - "--config"
+      - /app/config.yaml
+      - "--port"
+      - "4000"
+      - "--num_workers"
+      - "8"
+  postgres:
+    image: "postgres:16-alpine"
+    environment:
+      - POSTGRES_DB=${POSTGRES_DB:-litellm}
+      - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES
+      - POSTGRES_USER=$SERVICE_USER_POSTGRES
+    volumes:
+      - "pg-data:/var/lib/postgresql/data"
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - "pg_isready -h localhost -U ${POSTGRES_USER} -d ${POSTGRES_DB}"
+      interval: 5s
+      timeout: 5s
+      retries: 3
+  redis:
+    image: redis:7-alpine
+    command: redis-server --appendonly yes
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 5s
+      retries: 3