Source files / install/patches · raw

server-prompt-cache-keep.patch

diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index f6b27907a..c826dae92 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1641,7 +1641,9 @@ private:
                 }
 
                 // if we are about to lose a large portion of the existing context - save it in the prompt cache
-                if (f_keep < 0.5f) {
+                // with several slots, save it even if only the tail is lost: parallel requests with the same
+                // prefix (e.g. subagents with the root agent's system prompt) can then restore it in other slots
+                if (f_keep < 0.5f || (slots.size() > 1 && f_keep < 1.0f)) {
                     update_cache = true;
                 }
             }
@@ -1671,6 +1673,12 @@ private:
             }
         }
 
+        // a requested slot that is empty never passes the similarity check (f_keep = 0/0): still
+        // look in the prompt cache for it
+        if (ret && task.id_slot != -1 && ret->prompt.tokens.empty()) {
+            update_cache = true;
+        }
+
         if (ret) {
             update_cache = update_cache && prompt_cache;
 
@@ -1682,6 +1690,14 @@ private:
 
                 const int64_t t_start = ggml_time_us();
 
+                // idle slots are only saved after this task launched (cache_idle_slots): save them now so
+                // this slot can restore a prefix that another idle slot holds (no copy if already cached)
+                for (server_slot & slot : slots) {
+                    if (&slot != ret && !slot.is_processing()) {
+                        slot.prompt_save(*prompt_cache);
+                    }
+                }
+
                 ret->prompt_save(*prompt_cache);
 
                 if (!ret->prompt_load(*prompt_cache, task.tokens)) {
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 0d3beb313..4bb29db97 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1835,9 +1835,6 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
 
                 return false;
             }
-
-            data.clear();
-            data.shrink_to_fit();
         }
 
         {
@@ -1853,15 +1850,14 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
 
                     return false;
                 }
-
-                data.clear();
-                data.shrink_to_fit();
             }
         }
 
-        prompt = std::move(it_best->prompt);
+        // keep the entry: other slots (e.g. parallel subagents with the same system prompt) can
+        // restore the same prefix; mark it most recently used so it is evicted last
+        prompt = it_best->prompt.clone();
 
-        states.erase(it_best);
+        states.splice(states.end(), states, it_best);
     }
 
     return true;