/tags/ai/Recent content in Ai on OakestraHugoenCopyright (c) 2020-2024 ThuliteSat, 30 May 2026 16:27:22 +0200/blog/deploy-llms-and-agents-with-oakestra/Sat, 30 May 2026 16:27:22 +0200/blog/deploy-llms-and-agents-with-oakestra/<div class="expressive-code"> <figure class="frame not-content"> <figcaption class="header"> <span class="title"></span> </figcaption> <pre tabindex="0"><code>{ &#34;sla_version&#34;: &#34;v2.0&#34;, &#34;customerID&#34;: &#34;Admin&#34;, &#34;applications&#34;: [ { &#34;applicationID&#34;: &#34;&#34;, &#34;application_name&#34;: &#34;model&#34;, &#34;application_namespace&#34;: &#34;oakversity&#34;, &#34;application_desc&#34;: &#34;OpenClaw custom deployment&#34;, &#34;microservices&#34;: [ { &#34;microserviceID&#34;: &#34;&#34;, &#34;microservice_name&#34;: &#34;qwen3527b&#34;, &#34;microservice_namespace&#34;: &#34;oakmodel&#34;, &#34;virtualization&#34;: &#34;container&#34;, &#34;environment&#34;:[ &#34;OLLAMA_CONTEXT_LENGTH=128000&#34;, &#34;OLLAMA_FLASH_ATTENTION=1&#34;, &#34;OLLAMA_NUM_PARALLEL=4&#34;, &#34;OLLAMA_MAX_LOADED_MODELS=1&#34;, &#34;OLLAMA_KEEP_ALIVE=-1&#34; ], &#34;cmd&#34;: [&#34;/bin/sh&#34;,&#34;-c&#34;,&#34;ollama serve &amp; sleep 5; ollama run qwen3.5:27b --verbose; sleep inf&#34;], &#34;memory&#34;: 0, &#34;vcpus&#34;: 0, &#34;vgpus&#34;: 1, &#34;vtpus&#34;: 0, &#34;bandwidth_in&#34;: 0, &#34;bandwidth_out&#34;: 0, &#34;storage&#34;: 0, &#34;code&#34;: &#34;ghcr.io/giobart/ollama:v2&#34;, &#34;state&#34;: &#34;&#34;, &#34;port&#34;: &#34;&#34;, &#34;addresses&#34;:{ &#34;rr_ip&#34;: &#34;10.30.55.55&#34; }, &#34;added_files&#34;: [], &#34;constraints&#34;:[], &#34;volumes&#34;: [ { &#34;volume_id&#34;: &#34;my-named-volume&#34;, &#34;csi_driver&#34;: &#34;csi.oakestra.io/hostpath&#34;, &#34;mount_path&#34;: &#34;/root/.ollama&#34;, &#34;config&#34;: { &#34;host_path&#34;: &#34;ollama&#34; } } ] }, { &#34;microserviceID&#34;: &#34;&#34;, &#34;microservice_name&#34;: &#34;redis&#34;, &#34;microservice_namespace&#34;: &#34;dev&#34;, &#34;virtualization&#34;: &#34;container&#34;, &#34;environment&#34;:[], &#34;cmd&#34;: [], &#34;memory&#34;: 200, &#34;vcpus&#34;: 2, &#34;vgpus&#34;: 0, &#34;vtpus&#34;: 0, &#34;bandwidth_in&#34;: 0, &#34;bandwidth_out&#34;: 0, &#34;storage&#34;: 0, &#34;code&#34;: &#34;docker.io/library/redis:7-alpine&#34;, &#34;state&#34;: &#34;&#34;, &#34;port&#34;: &#34;&#34;, &#34;addresses&#34;:{ &#34;rr_ip&#34;: &#34;10.30.56.1&#34; }, &#34;added_files&#34;: [], &#34;constraints&#34;:[] }, { &#34;microserviceID&#34;: &#34;&#34;, &#34;microservice_name&#34;: &#34;litellm&#34;, &#34;microservice_namespace&#34;: &#34;oakbal&#34;, &#34;virtualization&#34;: &#34;container&#34;, &#34;environment&#34;:[&#34;LITELLM_DROP_PARAMS=True&#34;], &#34;cmd&#34;: [&#34;litellm&#34;,&#34;--config&#34;,&#34;/app/config.yaml&#34;], &#34;memory&#34;: 200, &#34;vcpus&#34;: 2, &#34;vgpus&#34;: 0, &#34;vtpus&#34;: 0, &#34;bandwidth_in&#34;: 0, &#34;bandwidth_out&#34;: 0, &#34;storage&#34;: 0, &#34;code&#34;: &#34;ghcr.io/berriai/litellm:main&#34;, &#34;state&#34;: &#34;&#34;, &#34;port&#34;: &#34;4000&#34;, &#34;addresses&#34;:{}, &#34;added_files&#34;: [], &#34;constraints&#34;:[], &#34;addresses&#34;:{ &#34;rr_ip&#34;: &#34;10.30.200.2&#34; }, &#34;volumes&#34;: [ { &#34;volume_id&#34;: &#34;litellm-volume&#34;, &#34;csi_driver&#34;: &#34;csi.oakestra.io/hostpath&#34;, &#34;mount_path&#34;: &#34;/app/&#34;, &#34;config&#34;: { &#34;host_path&#34;: &#34;litellm&#34; } } ], &#34;constraints&#34;:[ { &#34;type&#34;:&#34;direct&#34;, &#34;node&#34;:&#34;cm&#34;, &#34;cluster&#34;:&#34;defalut_cluster&#34; } ] }, { &#34;microserviceID&#34;: &#34;&#34;, &#34;microservice_name&#34;: &#34;searxng&#34;, &#34;microservice_namespace&#34;: &#34;dev&#34;, &#34;virtualization&#34;: &#34;container&#34;, &#34;cmd&#34;: [], &#34;memory&#34;: 200, &#34;vcpus&#34;: 1, &#34;vgpus&#34;: 0, &#34;vtpus&#34;: 0, &#34;bandwidth_in&#34;: 0, &#34;bandwidth_out&#34;: 0, &#34;storage&#34;: 0, &#34;code&#34;: &#34;docker.io/searxng/searxng:latest&#34;, &#34;state&#34;: &#34;&#34;, &#34;addresses&#34;:{ &#34;rr_ip&#34;: &#34;10.30.200.1&#34; }, &#34;added_files&#34;: [] } ] } ] }</code></pre> </figure> </div> <p>Add the configuration to the <code>/mnt/oakestra/hostpath/litellm/config.yaml</code></p>