From 0fa320992626753f1f28caef16954f371ab43491 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Tue, 29 Sep 2026 07:56:44 -0500 Subject: [PATCH 1/2] feat(model): bind local chat to engine context Publish per-slot limits to Assistant and request the selected answer reserve. Count the fully templated prompt through the owned llama engine before generation, with one deadline and cancellation path. Verified: provider, model-contract, and runtime test suites; Assistant shell and cancellation tests; source gates and affected-crate Clippy. --- capsules/assistant/browser/agent-context.js | 18 ++- capsules/assistant/browser/agent-live.js | 17 ++- capsules/assistant/browser/model-contract.js | 32 +++-- capsules/model-provider/src/adapters.rs | 87 ++++++++++- capsules/model-provider/src/config.rs | 41 ++++++ capsules/model-provider/src/contract.rs | 9 ++ capsules/model-provider/src/execution.rs | 135 ++++++++++++++++++ capsules/model-provider/src/journal.rs | 12 ++ capsules/model-provider/src/local_llama.rs | 2 + capsules/model-provider/src/state.rs | 1 + capsules/model-provider/src/test_support.rs | 34 ++++- docs/MODEL_PROVIDER.md | 10 ++ .../crates/elastos-model-contract/src/lib.rs | 21 ++- .../src/provider/local_model_broker.rs | 50 +++++-- scripts/home-agent-cancellation.test.mjs | 23 +-- scripts/home-agent-shell-smoke.mjs | 41 +++++- 16 files changed, 487 insertions(+), 46 deletions(-) diff --git a/capsules/assistant/browser/agent-context.js b/capsules/assistant/browser/agent-context.js index d6e9c03a9..dc260d9a0 100644 --- a/capsules/assistant/browser/agent-context.js +++ b/capsules/assistant/browser/agent-context.js @@ -84,7 +84,7 @@ export function resolveReasoningPolicy(requested, caps = TEXT_CONTRACT_CAPABILIT /** Char/4 estimator — not the provider tokenizer. Pack against a margin, not the raw cap. */ export const TOKEN_ESTIMATOR = "chars/4"; const ESTIMATOR_HEADROOM = 0.92; -const MIN_VIABLE_ESTIMATED_INPUT = 512; +const MIN_VIABLE_ESTIMATED_INPUT = 32; export class ContextOverflowError extends Error { constructor(message) { @@ -94,6 +94,21 @@ export class ContextOverflowError extends Error { } } +export function capabilitiesForOffer(offer, capabilities = TEXT_CONTRACT_CAPABILITIES) { + if (!offer?.context && offer?.routeKind !== "local") return capabilities; + const contextWindow = offer.context?.context_window_tokens; + const outputLimit = offer.context?.max_output_tokens; + if (!Number.isInteger(contextWindow) || contextWindow <= 0 || + !Number.isInteger(outputLimit) || outputLimit <= 0 || outputLimit >= contextWindow) { + throw new ContextOverflowError("Selected local model has incomplete context limits"); + } + return { ...capabilities, context: { + ...capabilities.context, + maxInputTokens: contextWindow, + maxOutputTokens: outputLimit, + } }; +} + export function approxTokens(text) { return Math.ceil(String(text || "").length / 4); } @@ -793,6 +808,7 @@ export function compileContext({ tokenEstimator: TOKEN_ESTIMATOR, estimatorHeadroom: tokenBudget.estimatorHeadroom, estimatedInputTokens: used, + outputReserve: tokenBudget.outputReserve, estimatedSystemTokens: systemTokens, estimatedRuntimeTokens: runtimeTokens, estimatedCurrentInputTokens: currentInputTokens, diff --git a/capsules/assistant/browser/agent-live.js b/capsules/assistant/browser/agent-live.js index 4ec5b337a..48814e5bb 100644 --- a/capsules/assistant/browser/agent-live.js +++ b/capsules/assistant/browser/agent-live.js @@ -18,6 +18,7 @@ import { } from "./model-contract.js"; import { compileContext, + capabilitiesForOffer, attachProviderPayload, splitSessionMessages, makeBudget, @@ -324,6 +325,8 @@ export function compileLiveContext({ degradedFallback = false, debug = false, } = {}) { + const offer = selectedLiveOffer(); + const effectiveCapabilities = capabilitiesForOffer(offer, capabilities); const raw = sessionMessages || session?.messages || []; let { history, currentInput: splitCurrent } = splitSessionMessages(raw); const current = currentInput || splitCurrent; @@ -365,12 +368,13 @@ export function compileLiveContext({ currentInput: current, constraints, intent: "", - budget: makeBudget(capabilities.context, clampLiveMaxTokens(maxTokens)), - capabilities, + budget: makeBudget(effectiveCapabilities.context, clampLiveMaxTokens(maxTokens)), + capabilities: effectiveCapabilities, transcriptChars: transcriptCharCount(raw), thinkingChars, }); attachProviderPayload(compiled); + compiled.manifest.offerId = offer?.offerId || ""; logContextManifest(compiled.manifest, compiled.invariants, { debug }); return compiled; } @@ -523,6 +527,10 @@ export async function streamChatViaContract( activeRun = null; throw contractError("no_model_offers", "no text model offer on this Home"); } + if (!turn.providerRunId && manifest.offerId && offer?.offerId !== manifest.offerId) { + activeRun = null; + throw contractError("model_selection_changed", "Selected model changed while context was prepared"); + } const resuming = Boolean(turn.providerRunId); let created; let runId; @@ -535,7 +543,10 @@ export async function streamChatViaContract( try { created = resuming ? await modelRunCall("runs_get", { run_id: turn.providerRunId, request_id: newRequestId() }) - : await modelRunCall("runs_create", textRunCreateBody({ offer, messages, requestId: createRequestId })); + : await modelRunCall("runs_create", textRunCreateBody({ + offer, messages, requestId: createRequestId, + maxOutputTokens: manifest.outputReserve, + })); runId = typeof created?.run_id === "string" ? created.run_id : ""; if (!runId || runId.trim() !== runId || runId.length > 80 || (resuming && runId !== turn.providerRunId)) { throw contractError("no_run_id", "contract returned no run id"); diff --git a/capsules/assistant/browser/model-contract.js b/capsules/assistant/browser/model-contract.js index c99acf066..4bdcffa92 100644 --- a/capsules/assistant/browser/model-contract.js +++ b/capsules/assistant/browser/model-contract.js @@ -285,7 +285,9 @@ export function textOfferRows(offers, backendReports = {}) { label: offer.title, detail: offerRowDetail(offer, facts), streamOutput: offer.stream_output === true, + routeKind: offerRouteKind(offer), inputSchemas: Array.isArray(offer.input_schemas) ? [...offer.input_schemas] : [], + context: offer.context && typeof offer.context === "object" ? { ...offer.context } : null, selectionFacts: facts, }; }); @@ -315,29 +317,33 @@ export function transcriptPrompt(messages) { return lines.join("\n\n"); } -export function textRunCreateBody({ offer, messages, requestId }) { +export function textRunCreateBody({ offer, messages, requestId, maxOutputTokens = null }) { if (!offer || typeof offer.offerId !== "string" || typeof offer.operation !== "string") { throw contractError("no_offer", "no text model offer selected"); } if (typeof requestId !== "string" || requestId.trim() === "") { throw contractError("no_request_id", "run request needs a request id"); } + const input = offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA) + ? { + schema: MODEL_TEXT_INPUT_V2_SCHEMA, + messages: messages.map(({ role, content }) => ({ + role: role === "agent" ? "assistant" : role, + content, + })), + } + : { + schema: MODEL_TEXT_INPUT_SCHEMA, + prompt: transcriptPrompt(messages), + }; + if (offer.context && Number.isInteger(maxOutputTokens) && maxOutputTokens > 0) { + input.max_output_tokens = maxOutputTokens; + } return { offer_id: offer.offerId, operation: offer.operation, request_id: requestId, - input: offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA) - ? { - schema: MODEL_TEXT_INPUT_V2_SCHEMA, - messages: messages.map(({ role, content }) => ({ - role: role === "agent" ? "assistant" : role, - content, - })), - } - : { - schema: MODEL_TEXT_INPUT_SCHEMA, - prompt: transcriptPrompt(messages), - }, + input, }; } diff --git a/capsules/model-provider/src/adapters.rs b/capsules/model-provider/src/adapters.rs index 7dd93bb61..eb8262ee4 100644 --- a/capsules/model-provider/src/adapters.rs +++ b/capsules/model-provider/src/adapters.rs @@ -476,6 +476,7 @@ enum LocalTextBackend { engine: LocalArtifactConfig, model: LocalArtifactConfig, settings: LocalLlamaSettings, + requested_output_tokens: Option, }, } @@ -1155,6 +1156,7 @@ impl AdapterExecutor for LiveAdapterExecutor { engine: engine.clone(), model: model.clone(), settings: settings.clone(), + requested_output_tokens: requested_local_output_tokens(input, settings)?, }, offer, binding, @@ -1327,6 +1329,28 @@ fn validate_text_messages( } } +fn requested_local_output_tokens( + input: &Value, + settings: &LocalLlamaSettings, +) -> std::result::Result, AdapterFault> { + let Some(value) = input.get("max_output_tokens") else { + return Ok(None); + }; + let requested = value.as_u64().ok_or_else(|| { + AdapterFault::context( + "model output request is invalid", + "max_output_tokens must be a positive integer", + ) + })?; + if requested == 0 || requested > u64::from(crate::config::local_output_token_limit(settings)) { + return Err(AdapterFault::context( + "model output request exceeds the profile", + "max_output_tokens exceeds local model limit", + )); + } + Ok(Some(requested as u32)) +} + pub(crate) fn is_local_text_backend_state(value: &Value) -> bool { value .get("schema") @@ -1684,7 +1708,7 @@ async fn run_local_text_worker_inner( ) .then(HostedBackendReport::default); let mut local_socket = None; - let (api_url, api_key, body, private_endpoint) = match &task.backend { + let (api_url, api_key, body, private_endpoint, context_window_tokens) = match &task.backend { LocalTextBackend::OpenRouterDecisions { api_url, api_key, @@ -1714,6 +1738,7 @@ async fn run_local_text_worker_inner( api_key.clone(), text_generation_request_body(&task.offer, model, &task.messages, None), false, + None, ), LocalTextBackend::OpenAiResponses { api_url, @@ -1736,6 +1761,7 @@ async fn run_local_text_worker_inner( })?, ), false, + None, ), LocalTextBackend::LocalLlama { engines, @@ -1743,6 +1769,7 @@ async fn run_local_text_worker_inner( engine, model, settings, + requested_output_tokens, } => { let endpoint = engines .endpoint_with_timeout( @@ -1754,14 +1781,22 @@ async fn run_local_text_worker_inner( ) .await .map_err(map_local_llama_fault)?; - let body = text_generation_request_body( + let mut body = text_generation_request_body( &task.offer, &endpoint.model, &task.messages, Some(endpoint.enable_thinking), ); + body["max_tokens"] = json!(requested_output_tokens + .unwrap_or_else(|| { crate::config::local_output_token_limit(settings) })); local_socket = endpoint.unix_socket; - (endpoint.api_url, None, body, true) + ( + endpoint.api_url, + None, + body, + true, + Some(settings.context_size / settings.parallel), + ) } }; let client = backend_client_with_socket( @@ -1776,6 +1811,52 @@ async fn run_local_text_worker_inner( } else { task.hosted_socket.as_deref() }; + if let Some(context_window_tokens) = context_window_tokens { + let count_url = format!("{api_url}/input_tokens"); + let count_request = client + .post(count_url) + .header("content-type", "application/json") + .json(&body); + let counted = tokio::select! { + _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()), + response = count_request.send() => response.map_err(|err| map_text_reqwest_failure(err, true))?, + }; + if !counted.status().is_success() { + return Err(map_effect_http_status(&counted, None)); + } + let count = tokio::select! { + _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()), + response = read_bounded_json_response_async(counted) => response?, + }; + let input_tokens = count["input_tokens"] + .as_u64() + .filter(|count| *count > 0) + .ok_or_else(|| { + AdapterFault::malformed( + "model backend returned invalid data", + "missing chat input token count", + ) + })?; + let output_tokens = body["max_tokens"].as_u64().unwrap_or(0); + if input_tokens.saturating_add(output_tokens) >= u64::from(context_window_tokens) { + return Err(AdapterFault::context( + "model context is full", + format!("templated input {input_tokens} plus output {output_tokens} exceeds context {context_window_tokens}"), + )); + } + } + // The count and generation share the one run deadline. + let client = if context_window_tokens.is_some() { + backend_client_with_socket( + remaining_run_timeout(task.deadline_ms)? + .as_millis() + .try_into() + .unwrap_or(u64::MAX), + local_socket.as_deref(), + )? + } else { + client + }; let request = { let effect = if matches!(&task.backend, LocalTextBackend::OpenAiResponses { .. }) { "responses" diff --git a/capsules/model-provider/src/config.rs b/capsules/model-provider/src/config.rs index 701530ffb..29fcff30c 100644 --- a/capsules/model-provider/src/config.rs +++ b/capsules/model-provider/src/config.rs @@ -412,6 +412,10 @@ impl LocalLlamaSettings { MAX_LOCAL_LLAMA_PARALLEL, "local llama parallel", )?; + anyhow::ensure!( + self.context_size / self.parallel >= 128, + "local llama context_size must provide at least 128 tokens per slot" + ); validate_nonzero_bounded(self.threads, MAX_LOCAL_LLAMA_THREADS, "local llama threads")?; validate_nonzero_bounded( self.batch_threads, @@ -579,6 +583,15 @@ impl ConfiguredOffer { output_modalities: self.output_modalities.clone(), stream_output: self.adapter.stream_output(), policy: self.policy.summary(), + context: match &self.adapter { + AdapterConfig::LocalLlamaCppText { settings, .. } => { + Some(crate::contract::LocalContextLimits { + context_window_tokens: settings.context_size / settings.parallel, + max_output_tokens: local_output_token_limit(settings), + }) + } + _ => None, + }, hosted, } } @@ -588,6 +601,7 @@ impl ConfiguredOffer { pub fn execution_summary(&self) -> OfferSummary { let mut summary = self.summary(); summary.input_schemas.clear(); + summary.context = None; summary } @@ -711,6 +725,12 @@ impl ConfiguredOffer { } } +pub(crate) fn local_output_token_limit(settings: &LocalLlamaSettings) -> u32 { + // Keep a conservative answer slice in each server slot. Output bytes have + // their own independent transport bound. + (settings.context_size / settings.parallel / 4).clamp(1, 1_024) +} + pub fn journal_root(base_path: &str, configured: Option<&str>) -> Result { if let Some(configured) = configured { validate_bounded_trimmed(configured, "journal_dir", MAX_JOURNAL_DIR_BYTES)?; @@ -1261,6 +1281,27 @@ mod tests { assert_ne!(responses.execution_binding_hash().unwrap(), legacy_hash); } + #[test] + fn local_context_summary_uses_one_slot_and_rejects_unusable_profiles() { + let mut offer = local_llama_offer(Path::new("/tmp/engine"), Path::new("/tmp/model")); + let original_revision = offer.execution_binding_hash().unwrap(); + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 4_096); + assert_eq!(limits.max_output_tokens, 1_024); + if let AdapterConfig::LocalLlamaCppText { settings, .. } = &mut offer.adapter { + settings.parallel = 16; + assert_eq!(settings.context_size / settings.parallel, 256); + settings.validate().unwrap(); + settings.parallel = 64; + assert!(settings.validate().is_err()); + settings.parallel = 16; + } + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 256); + assert_eq!(limits.max_output_tokens, 64); + assert_ne!(offer.execution_binding_hash().unwrap(), original_revision); + } + #[test] fn execution_binding_hash_changes_only_for_semantic_execution_inputs() { let openai = base_offer(); diff --git a/capsules/model-provider/src/contract.rs b/capsules/model-provider/src/contract.rs index 57ba3e7ed..15e021272 100644 --- a/capsules/model-provider/src/contract.rs +++ b/capsules/model-provider/src/contract.rs @@ -245,9 +245,18 @@ pub struct OfferSummary { pub stream_output: bool, pub policy: OfferPolicySummary, #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub hosted: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LocalContextLimits { + pub context_window_tokens: u32, + pub max_output_tokens: u32, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct HostedOfferDisclosure { diff --git a/capsules/model-provider/src/execution.rs b/capsules/model-provider/src/execution.rs index c5e19d4da..06d8263c3 100644 --- a/capsules/model-provider/src/execution.rs +++ b/capsules/model-provider/src/execution.rs @@ -3233,6 +3233,141 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn local_llama_counts_template_before_generation_and_bounds_output() { + let root = temp_root("local-llama-context-budget"); + let (offer, events, root) = local_llama_offer(&root, "healthy"); + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 256); + assert_eq!(limits.max_output_tokens, 64); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + + let accepted_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [ + {"role":"system","content":"Keep the roles."}, + {"role":"user","content":"日本語 🌎"}, + ], + "max_output_tokens": 32, + }); + let accepted_binding = create_binding("request:budget-unicode", &offer, &accepted_input); + let accepted = create_run(&provider, &offer, &accepted_binding, &accepted_input); + let accepted_run = accepted["data"]["run_id"].as_str().unwrap(); + let terminal = + wait_for_terminal(&provider, accepted_run, &access_binding(&accepted_binding)); + assert_eq!(terminal["data"]["status"], "completed", "{terminal:#}"); + let first_events = fake_llama_events(&events); + let count_index = first_events + .iter() + .position(|line| line.starts_with("count:")) + .unwrap(); + let request_index = first_events + .iter() + .position(|line| line == "request:Keep the roles.") + .unwrap(); + assert!(count_index < request_index); + assert!(first_events.iter().any(|line| line == "max_tokens:32")); + let prompt_tokens: u32 = first_events[count_index]["count:".len()..].parse().unwrap(); + assert!(prompt_tokens + 32 < limits.context_window_tokens); + + let long_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"x".repeat(200)}], + "max_output_tokens": 64, + }); + let long_binding = create_binding("request:budget-long", &offer, &long_input); + let long_run = create_run(&provider, &offer, &long_binding, &long_input); + let long_terminal = wait_for_terminal( + &provider, + long_run["data"]["run_id"].as_str().unwrap(), + &access_binding(&long_binding), + ); + assert_eq!( + long_terminal["data"]["terminal"]["error"]["code"], + "context_rejected" + ); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 1 + ); + + let excessive_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"short"}], + "max_output_tokens": 65, + }); + let excessive_binding = + create_binding("request:budget-excessive", &offer, &excessive_input); + let excessive = create_run(&provider, &offer, &excessive_binding, &excessive_input); + assert_eq!( + excessive["data"]["terminal"]["error"]["code"], + "context_rejected" + ); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 1 + ); + provider.shutdown_on_eof(); + } + + #[cfg(unix)] + #[test] + fn local_llama_missing_count_fails_before_generation() { + let root = temp_root("local-llama-missing-count"); + let (offer, events, root) = local_llama_offer(&root, "incomplete_count"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let input = text_input("hello"); + let binding = create_binding("request:missing-count", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert_eq!(terminal["data"]["status"], "failed"); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 0 + ); + provider.shutdown_on_eof(); + } + + #[cfg(unix)] + #[test] + fn local_llama_stop_interrupts_stalled_count_body() { + let root = temp_root("local-llama-stalled-count"); + let (offer, events, root) = local_llama_offer(&root, "stalled_count"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let input = text_input("hello"); + let binding = create_binding("request:stalled-count", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + wait_for_fake_llama_event(&events, "count_body_started"); + let started = Instant::now(); + let _ = cancel_run(&provider, run_id, &access_binding(&binding)); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(terminal["data"]["status"], "settlement_unknown"); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 0 + ); + provider.shutdown_on_eof(); + } + #[cfg(unix)] #[test] fn local_llama_streams_concurrent_runs_reuses_child_and_guards_refresh() { diff --git a/capsules/model-provider/src/journal.rs b/capsules/model-provider/src/journal.rs index 3fb53f0f8..dfbf1d1df 100644 --- a/capsules/model-provider/src/journal.rs +++ b/capsules/model-provider/src/journal.rs @@ -517,6 +517,18 @@ fn validate_stored_offer(path: &Path, offer: &OfferSummary) -> Result<(), Provid )?; validate_modalities(path, "input", &offer.input_modalities)?; validate_modalities(path, "output", &offer.output_modalities)?; + if let Some(context) = &offer.context { + if context.context_window_tokens < 128 + || context.context_window_tokens > crate::config::MAX_LOCAL_LLAMA_CONTEXT_SIZE + || context.max_output_tokens == 0 + || context.max_output_tokens >= context.context_window_tokens + { + return Err(ProviderFault::corrupt_journal(format!( + "model run journal context limit is invalid at {}", + path.display() + ))); + } + } if let Some(hosted) = offer.hosted.as_ref() { if hosted.placement != HOSTED_PLACEMENT || hosted.selection_mode != HOSTED_SELECTION_PINNED diff --git a/capsules/model-provider/src/local_llama.rs b/capsules/model-provider/src/local_llama.rs index 38a89b011..e43df4023 100644 --- a/capsules/model-provider/src/local_llama.rs +++ b/capsules/model-provider/src/local_llama.rs @@ -367,6 +367,7 @@ fn engine_arguments( args.extend([ "--ctx-size".into(), settings.context_size.to_string().into(), + "--no-context-shift".into(), "--parallel".into(), settings.parallel.to_string().into(), "--threads".into(), @@ -952,6 +953,7 @@ mod tests { "11434", "--ctx-size", "256", + "--no-context-shift", "--parallel", "2", "--threads", diff --git a/capsules/model-provider/src/state.rs b/capsules/model-provider/src/state.rs index 0a1e6e3b9..83823a1c7 100644 --- a/capsules/model-provider/src/state.rs +++ b/capsules/model-provider/src/state.rs @@ -907,6 +907,7 @@ impl ModelProviderState { let current_summary = offer.execution_summary(); let mut recorded_summary = run.offer.clone(); recorded_summary.input_schemas.clear(); + recorded_summary.context = None; let current_hash = offer.execution_binding_hash().map_err(|err| { ProviderFault::internal(format!( "failed to derive model execution binding hash: {err}" diff --git a/capsules/model-provider/src/test_support.rs b/capsules/model-provider/src/test_support.rs index f60880234..9cca2c473 100644 --- a/capsules/model-provider/src/test_support.rs +++ b/capsules/model-provider/src/test_support.rs @@ -163,7 +163,7 @@ class Handler(http.server.BaseHTTPRequestHandler): self.wfile.write(json.dumps(body).encode('utf-8')) def do_POST(self): - if self.path != '/v1/chat/completions': + if self.path not in ('/v1/chat/completions', '/v1/chat/completions/input_tokens'): self.send_error(404) return length = int(self.headers.get('Content-Length', '0')) @@ -172,10 +172,42 @@ class Handler(http.server.BaseHTTPRequestHandler): if body.get('model') != alias or body.get('chat_template_kwargs') != {'enable_thinking': False}: self.send_error(400) return + if self.path == '/v1/chat/completions/input_tokens': + # A fixture tokenizer charges UTF-8 content and per-message template + # markers. The pinned engine's endpoint owns exact production counts. + count = sum(len(message['content'].encode('utf-8')) + 8 for message in body['messages']) + record('count:' + str(count)) + response = {'object': 'response.input_tokens', 'input_tokens': count} + if mode == 'incomplete_count': + del response['input_tokens'] + if mode == 'stalled_count': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', '100') + self.end_headers() + self.wfile.write(b'{') + self.wfile.flush() + record('count_body_started') + while True: + try: + self.wfile.write(b' ') + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + record('count_disconnected') + return + time.sleep(0.025) + data = json.dumps(response).encode('utf-8') + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + return active = model_path.with_suffix('.active') if mode == 'active_after_disconnect' and prompt == 'stall': active.touch() record('request:' + prompt) + record('max_tokens:' + str(body.get('max_tokens'))) if mode == 'crash_once' and prompt == 'crash': marker = model_path.with_suffix('.crashed') if not marker.exists(): diff --git a/docs/MODEL_PROVIDER.md b/docs/MODEL_PROVIDER.md index f07c114f0..89759bd6c 100644 --- a/docs/MODEL_PROVIDER.md +++ b/docs/MODEL_PROVIDER.md @@ -141,6 +141,16 @@ The provider binds these facts before dispatch: Secrets, raw provider credentials, and private endpoint details do not belong in the session journal or App-visible result. +For a local llama offer, `offers_list` includes `context.context_window_tokens` +for one engine slot and `context.max_output_tokens` for the selected profile. +Assistant packs against these values. A text/v1 or text/v2 input can include +`max_output_tokens`; Runtime binds that value in the input hash. An omitted +value uses the profile limit. Before generation, the provider asks the pinned +engine to count the exact chat request with its template and special tokens. +It sends generation only when the counted input and requested answer fit the +slot. The offer's input and output byte policies remain separate transport +bounds. + ## Stream lifecycle A model request follows one observable lifecycle: diff --git a/elastos/crates/elastos-model-contract/src/lib.rs b/elastos/crates/elastos-model-contract/src/lib.rs index e6d218106..4b7299491 100644 --- a/elastos/crates/elastos-model-contract/src/lib.rs +++ b/elastos/crates/elastos-model-contract/src/lib.rs @@ -163,7 +163,13 @@ pub fn model_input_hash(input: &Value) -> ContractResult { pub fn validate_text_input_v2(input: &Value) -> ContractResult<&[Value]> { let invalid = || ContractError::new("invalid text/v2 conversation"); let object = input.as_object().ok_or_else(invalid)?; - if object.len() != 2 || input["schema"] != TEXT_INPUT_V2_SCHEMA { + if (object.len() != 2 && object.len() != 3) + || input["schema"] != TEXT_INPUT_V2_SCHEMA + || (object.len() == 3 + && input["max_output_tokens"] + .as_u64() + .is_none_or(|count| count == 0 || count > u32::MAX as u64)) + { return Err(invalid()); } let messages = input["messages"].as_array().ok_or_else(invalid)?; @@ -287,6 +293,19 @@ mod tests { json!(vec![user.clone(); MAX_TEXT_MESSAGES + 1]), ]; assert!(validate_text_input_v2(&extra_top).is_err()); + let bounded = + json!({"schema":TEXT_INPUT_V2_SCHEMA,"messages":[user.clone()],"max_output_tokens":32}); + assert!(validate_text_input_v2(&bounded).is_ok()); + for invalid_limit in [ + json!(0), + json!(-1), + json!("32"), + json!(u64::from(u32::MAX) + 1), + ] { + let mut invalid = bounded.clone(); + invalid["max_output_tokens"] = invalid_limit; + assert!(validate_text_input_v2(&invalid).is_err()); + } assert!( validate_text_input_v2(&json!({"schema":TEXT_INPUT_V1_SCHEMA,"messages":[user]})) .is_err() diff --git a/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs b/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs index 478838e90..5538f5c2c 100644 --- a/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs +++ b/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs @@ -102,7 +102,9 @@ async fn forward( let route = &request[..line_end]; if !matches!( route, - b"GET /v1/models HTTP/1.1" | b"POST /v1/chat/completions HTTP/1.1" + b"GET /v1/models HTTP/1.1" + | b"POST /v1/chat/completions HTTP/1.1" + | b"POST /v1/chat/completions/input_tokens HTTP/1.1" ) { return Err(io::Error::new( io::ErrorKind::PermissionDenied, @@ -176,7 +178,11 @@ fn request_body_length(headers: &[u8], route: &[u8]) -> io::Result { let length = length.unwrap_or(0); if length > MAX_REQUEST_BODY_BYTES || (route == b"GET /v1/models HTTP/1.1" && length != 0) - || (route == b"POST /v1/chat/completions HTTP/1.1" && length == 0) + || (matches!( + route, + b"POST /v1/chat/completions HTTP/1.1" + | b"POST /v1/chat/completions/input_tokens HTTP/1.1" + ) && length == 0) { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -310,7 +316,9 @@ mod tests { let engine = dir.path().join("engine.sock"); let mut child = tokio::process::Command::new("/usr/bin/python3") .arg("-c") - .arg(r"import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(1); c,_=s.accept(); c.recv(4096); c.sendall(b'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok'); c.close()") + .arg(r"import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(2) +for _ in range(2): + c,_=s.accept(); request=c.recv(4096); body=b'ok' if request.startswith(b'GET /v1/models ') else b'counted' if request.startswith(b'POST /v1/chat/completions/input_tokens ') else b'wrong'; c.sendall(b'HTTP/1.1 200 OK\r\nContent-Length: '+str(len(body)).encode()+b'\r\nConnection: close\r\n\r\n'+body); c.close()") .arg(&engine) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -351,6 +359,20 @@ mod tests { .unwrap() .unwrap(); assert!(response.ends_with(b"\r\n\r\nok")); + let mut count = UnixStream::connect(&broker).await.unwrap(); + count + .write_all( + b"POST /v1/chat/completions/input_tokens HTTP/1.1\r\nContent-Length: 2\r\n\r\n{}", + ) + .await + .unwrap(); + count.shutdown().await.unwrap(); + let mut counted = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), count.read_to_end(&mut counted)) + .await + .unwrap() + .unwrap(); + assert!(counted.ends_with(b"\r\n\r\ncounted")); assert!(child.wait().await.unwrap().success()); task.abort(); } @@ -375,15 +397,19 @@ mod tests { #[test] fn one_request_body_is_bounded_and_chunked_or_duplicate_lengths_are_refused() { - let route = b"POST /v1/chat/completions HTTP/1.1"; - assert_eq!( - request_body_length( - b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 2\r\n\r\n", - route - ) - .unwrap(), - 2 - ); + for route in [ + b"POST /v1/chat/completions HTTP/1.1".as_slice(), + b"POST /v1/chat/completions/input_tokens HTTP/1.1".as_slice(), + ] { + let header = format!( + "{}\r\nContent-Length: 2\r\n\r\n", + std::str::from_utf8(route).unwrap() + ); + assert_eq!(request_body_length(header.as_bytes(), route).unwrap(), 2); + assert!(request_body_length(b"Content-Length: 0\r\n\r\n", route).is_err()); + assert!(request_body_length(b"Content-Length: 4194305\r\n\r\n", route).is_err()); + } + let route = b"POST /v1/chat/completions/input_tokens HTTP/1.1"; for header in [ b"POST /v1/chat/completions HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n".as_slice(), b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 2\r\nContent-Length: 3\r\n\r\n", diff --git a/scripts/home-agent-cancellation.test.mjs b/scripts/home-agent-cancellation.test.mjs index da5a372be..ec5a82272 100644 --- a/scripts/home-agent-cancellation.test.mjs +++ b/scripts/home-agent-cancellation.test.mjs @@ -24,6 +24,7 @@ const workspace = await import("../capsules/assistant/browser/agent-workspace.js const sessions = await import("../capsules/assistant/browser/agent-sessions.js"); const defer = () => { let resolve, reject; const promise = new Promise((r, e) => { resolve = r; reject = e; }); return { promise, resolve, reject }; }; const terminal = (kind) => ({ events: [{ sequence: 1, kind, terminal: true, data: kind === "output" ? { schema: "elastos.model.output.text/v1", text: "Done" } : {} }], next_cursor: 1, has_more: false }); +const fixtureContext = { context_window_tokens: 4096, max_output_tokens: 1024 }; test("content intent survives missing catalog and overlapping refresh; hosted replacement is deliberate", async t => { t.after(() => live.selectLiveOffer("")); @@ -307,7 +308,7 @@ async function fixture(cancelReject = false, saved = null) { const body = JSON.parse(init.body); calls.push({ op, body }); let data; - if (op === "offers_list") data = { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }] }; + if (op === "offers_list") data = { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }] }; else if (op === "runs_create") data = { run_id: "run-fixture", sequence_cursor: 0 }; else if (op === "runs_get") data = { run_id: "run-fixture", status: "running" }; else if (op === "runs_events") data = await events.promise; @@ -375,7 +376,7 @@ test("an immediately failed create settles without polling events", async () => const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-full", sequence_cursor: 0, status: "failed", @@ -397,7 +398,7 @@ test("a retained unknown create settles from the create view without polling", a const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-pruned", sequence_cursor: 0, status: "settlement_unknown", @@ -447,7 +448,7 @@ test("an attached poll at cursor 7 settles from a retained terminal event", asyn const body = init?.body ? JSON.parse(init.body) : null; calls.push({ op, body }); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-attached", sequence_cursor: 7, status: "running", @@ -477,7 +478,7 @@ test("a completed pruned journal states that live output was not retained", asyn const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-completed-pruned", sequence_cursor: 0, status: "completed", @@ -504,7 +505,7 @@ test("an attached poll of a completed pruned journal states that output was not const body = init?.body ? JSON.parse(init.body) : null; calls.push({ op, body }); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-completed-events", sequence_cursor: 2, status: "running", @@ -609,7 +610,7 @@ for (const mode of ["lost_response", "invalid_json", "missing_run_id", "invalid_ globalThis.fetch = async (url, init) => { const op = new URL(url).pathname.split("/").pop(); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; assert.equal(op, "runs_create"); const body = JSON.parse(init.body); @@ -794,7 +795,7 @@ async function runUnretainedLiveTurn(t, { create, eventsPages = [] }) { if (op === "offers_list") { offerLists += 1; return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; } if (op === "runs_create") return { ok: true, json: async () => create }; @@ -889,7 +890,7 @@ test("immediate failed create shows the provider result and restores it after sa const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-overflow", sequence_cursor: 2, status: "failed", @@ -1115,7 +1116,7 @@ test("actual composer preserves draft for missing runs and unknown acceptance un calls.push({ op, body: JSON.parse(init.body) }); if (op === "runs_get") return { ok: false, status: 404, json: async () => ({ code: "run_not_found", message: "model run not found" }) }; return { ok: true, json: async () => op === "offers_list" - ? { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }] } + ? { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }] } : op === "runs_create" ? { run_id: "new-draft-run", sequence_cursor: 0 } : terminal("settlement_unknown") }; }; const draft = { text: "new unsent prompt", parts: [{ id: "attachment", kind: "file", name: "notes.txt", text: "reference" }] }; @@ -1182,7 +1183,7 @@ test("only a matching fresh invocation refusal settles a failed create", async ( globalThis.fetch = async (url, init) => { const op = new URL(url).pathname.split("/").pop(); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: offerId, title: "Fixture", operation: "text.generate", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: offerId, title: "Fixture", operation: "text.generate", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op !== "runs_create") throw new Error(`unexpected ${op}`); const body = JSON.parse(init.body); calls.push(body); diff --git a/scripts/home-agent-shell-smoke.mjs b/scripts/home-agent-shell-smoke.mjs index b75b22961..1f5a56aaa 100644 --- a/scripts/home-agent-shell-smoke.mjs +++ b/scripts/home-agent-shell-smoke.mjs @@ -68,7 +68,7 @@ assert.ok( "Assistant Chat and Studio decode events through the typed contract module", ); assert.ok(agentLive.includes("after_sequence: afterSequence"), "runs_events is polled by after_sequence"); -assert.ok(agentLive.includes("textRunCreateBody({ offer, messages, requestId: createRequestId })")); +assert.ok(agentLive.includes("maxOutputTokens: manifest.outputReserve")); assert.ok(agentLive.includes('modelRunCall("runs_cancel", { run_id: runId, request_id: newRequestId() })')); assert.ok(!agentLive.includes("agent-run-cursor"), "the URUX cursor helper is gone"); @@ -236,6 +236,7 @@ assert.ok(!/Planning weekend|calm weekend/.test(read("capsules/assistant/browser /* ---- the pure contract module -------------------------------------------- */ const contract = await import(new URL("model-contract.js", capsuleDir)); +const context = await import(new URL("agent-context.js", capsuleDir)); const homeMessageContract = await import( new URL("capsules/home-gui/browser/home-agent-message-contract.js", root) ); @@ -356,7 +357,9 @@ assert.deepEqual(rows, [ label: "Local chat", detail: "This Home · local", streamOutput: true, + routeKind: "local", inputSchemas: [], + context: null, selectionFacts: localFacts, }, ]); @@ -502,6 +505,42 @@ assert.deepEqual(body, { request_id: "req-1", input: { schema: "elastos.model.input.text/v1", prompt: contract.transcriptPrompt(messages) }, }); +const boundedLocal = contract.textOfferRows([{ ...textOffers[0], context: { + context_window_tokens: 256, max_output_tokens: 64, +} }])[0]; +const effective = context.capabilitiesForOffer(boundedLocal); +assert.deepEqual(effective.context, { + ...context.TEXT_CONTRACT_CAPABILITIES.context, + maxInputTokens: 256, + maxOutputTokens: 64, +}); +assert.equal(context.makeBudget(effective.context, 32).outputReserve, 32); +assert.ok(context.makeBudget(effective.context, 32).hardEstimatedInputLimit < 256 - 32); +const longHistory = Array.from({ length: 25 }, (_, i) => ({ + id: `turn-${i}`, role: i % 2 ? "assistant" : "user", content: `turn ${i} ${"x".repeat(400)}`, +})); +const packed = context.compileContext({ + history: longHistory, + currentInput: { id: "current", role: "user", content: "Continue" }, + constraints: [{ id: "system", kind: "system_policy", role: "system", content: "Be brief.", mustInclude: true }], + budget: context.makeBudget(effective.context, 32), + capabilities: effective, +}); +assert.ok(packed.omitted.length > 0); +assert.ok(packed.manifest.estimatedInputTokens <= packed.budget.hardEstimatedInputLimit); +assert.equal(packed.manifest.outputReserve, 32); +for (const missing of [null, {}, { context_window_tokens: 256 }, + { context_window_tokens: 256, max_output_tokens: 256 }]) { + assert.throws(() => context.capabilitiesForOffer({ ...boundedLocal, context: missing }), + { code: "context_overflow" }); +} +assert.deepEqual(contract.textRunCreateBody({ + offer: boundedLocal, messages, requestId: "bounded", maxOutputTokens: 32, +}).input, { + schema: contract.MODEL_TEXT_INPUT_SCHEMA, + prompt: contract.transcriptPrompt(messages), + max_output_tokens: 32, +}); // Follow the serialized offer response through selection into the run request. const conversation = [ { role: "system", content: " Keep spacing. " }, From 07d525c46158c166417dea32e82afbd88276e9d5 Mon Sep 17 00:00:00 2001 From: Anders Alm Date: Wed, 30 Sep 2026 06:28:57 -0500 Subject: [PATCH 2/2] fix(model): reject hosted output token requests --- capsules/model-provider/src/adapters.rs | 8 +++++ capsules/model-provider/src/execution.rs | 39 ++++++++++++++++++++++++ docs/MODEL_PROVIDER.md | 7 +++-- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/capsules/model-provider/src/adapters.rs b/capsules/model-provider/src/adapters.rs index eb8262ee4..b026cd5a0 100644 --- a/capsules/model-provider/src/adapters.rs +++ b/capsules/model-provider/src/adapters.rs @@ -1300,6 +1300,14 @@ fn validate_text_messages( offer: &ConfiguredOffer, input: &Value, ) -> std::result::Result, AdapterFault> { + if input.get("max_output_tokens").is_some() + && !matches!(offer.adapter, AdapterConfig::LocalLlamaCppText { .. }) + { + return Err(AdapterFault::context( + "model output request is not supported by this offer", + "max_output_tokens is supported only by local llama offers", + )); + } let invalid = || { AdapterFault::context( "model input is invalid", diff --git a/capsules/model-provider/src/execution.rs b/capsules/model-provider/src/execution.rs index 06d8263c3..9a2ba234f 100644 --- a/capsules/model-provider/src/execution.rs +++ b/capsules/model-provider/src/execution.rs @@ -3233,6 +3233,45 @@ mod tests { ); } + #[test] + fn hosted_text_rejects_requested_output_limit_before_dispatch() { + let server = start_server(vec![]); + let v1 = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V1_SCHEMA, + "prompt": "hello", + "max_output_tokens": 64, + }); + let v2 = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"hello"}], + "max_output_tokens": 64, + }); + for (offer, input) in [ + (local_text_offer(&server.base_url), v1.clone()), + (local_text_offer(&server.base_url), v2), + (responses_text_offer(&server.base_url), v1), + ] { + let root = temp_root("hosted-output-limit"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let binding = create_binding("request:hosted-output-limit", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert_eq!(terminal["data"]["status"], "failed", "{terminal:#}"); + assert_eq!( + terminal["data"]["terminal"]["error"]["code"], "context_rejected", + "{terminal:#}" + ); + assert_eq!( + terminal["data"]["terminal"]["error"]["message"], + "model output request is not supported by this offer" + ); + provider.shutdown_on_eof(); + } + assert!(server.requests.lock().unwrap().is_empty()); + } + #[cfg(unix)] #[test] fn local_llama_counts_template_before_generation_and_bounds_output() { diff --git a/docs/MODEL_PROVIDER.md b/docs/MODEL_PROVIDER.md index 89759bd6c..88888e239 100644 --- a/docs/MODEL_PROVIDER.md +++ b/docs/MODEL_PROVIDER.md @@ -143,9 +143,10 @@ the session journal or App-visible result. For a local llama offer, `offers_list` includes `context.context_window_tokens` for one engine slot and `context.max_output_tokens` for the selected profile. -Assistant packs against these values. A text/v1 or text/v2 input can include -`max_output_tokens`; Runtime binds that value in the input hash. An omitted -value uses the profile limit. Before generation, the provider asks the pinned +Assistant packs against these values. For local llama offers only, a text/v1 or +text/v2 input can include `max_output_tokens`; Runtime binds that value in the +input hash. Hosted offers reject this field with `context_rejected`. An omitted +value uses the local profile limit. Before generation, the provider asks the pinned engine to count the exact chat request with its template and special tokens. It sends generation only when the counted input and requested answer fit the slot. The offer's input and output byte policies remain separate transport