diff --git a/capsules/assistant/browser/agent-context.js b/capsules/assistant/browser/agent-context.js index d6e9c03a9..dc260d9a0 100644 --- a/capsules/assistant/browser/agent-context.js +++ b/capsules/assistant/browser/agent-context.js @@ -84,7 +84,7 @@ export function resolveReasoningPolicy(requested, caps = TEXT_CONTRACT_CAPABILIT /** Char/4 estimator — not the provider tokenizer. Pack against a margin, not the raw cap. */ export const TOKEN_ESTIMATOR = "chars/4"; const ESTIMATOR_HEADROOM = 0.92; -const MIN_VIABLE_ESTIMATED_INPUT = 512; +const MIN_VIABLE_ESTIMATED_INPUT = 32; export class ContextOverflowError extends Error { constructor(message) { @@ -94,6 +94,21 @@ export class ContextOverflowError extends Error { } } +export function capabilitiesForOffer(offer, capabilities = TEXT_CONTRACT_CAPABILITIES) { + if (!offer?.context && offer?.routeKind !== "local") return capabilities; + const contextWindow = offer.context?.context_window_tokens; + const outputLimit = offer.context?.max_output_tokens; + if (!Number.isInteger(contextWindow) || contextWindow <= 0 || + !Number.isInteger(outputLimit) || outputLimit <= 0 || outputLimit >= contextWindow) { + throw new ContextOverflowError("Selected local model has incomplete context limits"); + } + return { ...capabilities, context: { + ...capabilities.context, + maxInputTokens: contextWindow, + maxOutputTokens: outputLimit, + } }; +} + export function approxTokens(text) { return Math.ceil(String(text || "").length / 4); } @@ -793,6 +808,7 @@ export function compileContext({ tokenEstimator: TOKEN_ESTIMATOR, estimatorHeadroom: tokenBudget.estimatorHeadroom, estimatedInputTokens: used, + outputReserve: tokenBudget.outputReserve, estimatedSystemTokens: systemTokens, estimatedRuntimeTokens: runtimeTokens, estimatedCurrentInputTokens: currentInputTokens, diff --git a/capsules/assistant/browser/agent-live.js b/capsules/assistant/browser/agent-live.js index 4ec5b337a..48814e5bb 100644 --- a/capsules/assistant/browser/agent-live.js +++ b/capsules/assistant/browser/agent-live.js @@ -18,6 +18,7 @@ import { } from "./model-contract.js"; import { compileContext, + capabilitiesForOffer, attachProviderPayload, splitSessionMessages, makeBudget, @@ -324,6 +325,8 @@ export function compileLiveContext({ degradedFallback = false, debug = false, } = {}) { + const offer = selectedLiveOffer(); + const effectiveCapabilities = capabilitiesForOffer(offer, capabilities); const raw = sessionMessages || session?.messages || []; let { history, currentInput: splitCurrent } = splitSessionMessages(raw); const current = currentInput || splitCurrent; @@ -365,12 +368,13 @@ export function compileLiveContext({ currentInput: current, constraints, intent: "", - budget: makeBudget(capabilities.context, clampLiveMaxTokens(maxTokens)), - capabilities, + budget: makeBudget(effectiveCapabilities.context, clampLiveMaxTokens(maxTokens)), + capabilities: effectiveCapabilities, transcriptChars: transcriptCharCount(raw), thinkingChars, }); attachProviderPayload(compiled); + compiled.manifest.offerId = offer?.offerId || ""; logContextManifest(compiled.manifest, compiled.invariants, { debug }); return compiled; } @@ -523,6 +527,10 @@ export async function streamChatViaContract( activeRun = null; throw contractError("no_model_offers", "no text model offer on this Home"); } + if (!turn.providerRunId && manifest.offerId && offer?.offerId !== manifest.offerId) { + activeRun = null; + throw contractError("model_selection_changed", "Selected model changed while context was prepared"); + } const resuming = Boolean(turn.providerRunId); let created; let runId; @@ -535,7 +543,10 @@ export async function streamChatViaContract( try { created = resuming ? await modelRunCall("runs_get", { run_id: turn.providerRunId, request_id: newRequestId() }) - : await modelRunCall("runs_create", textRunCreateBody({ offer, messages, requestId: createRequestId })); + : await modelRunCall("runs_create", textRunCreateBody({ + offer, messages, requestId: createRequestId, + maxOutputTokens: manifest.outputReserve, + })); runId = typeof created?.run_id === "string" ? created.run_id : ""; if (!runId || runId.trim() !== runId || runId.length > 80 || (resuming && runId !== turn.providerRunId)) { throw contractError("no_run_id", "contract returned no run id"); diff --git a/capsules/assistant/browser/model-contract.js b/capsules/assistant/browser/model-contract.js index c99acf066..4bdcffa92 100644 --- a/capsules/assistant/browser/model-contract.js +++ b/capsules/assistant/browser/model-contract.js @@ -285,7 +285,9 @@ export function textOfferRows(offers, backendReports = {}) { label: offer.title, detail: offerRowDetail(offer, facts), streamOutput: offer.stream_output === true, + routeKind: offerRouteKind(offer), inputSchemas: Array.isArray(offer.input_schemas) ? [...offer.input_schemas] : [], + context: offer.context && typeof offer.context === "object" ? { ...offer.context } : null, selectionFacts: facts, }; }); @@ -315,29 +317,33 @@ export function transcriptPrompt(messages) { return lines.join("\n\n"); } -export function textRunCreateBody({ offer, messages, requestId }) { +export function textRunCreateBody({ offer, messages, requestId, maxOutputTokens = null }) { if (!offer || typeof offer.offerId !== "string" || typeof offer.operation !== "string") { throw contractError("no_offer", "no text model offer selected"); } if (typeof requestId !== "string" || requestId.trim() === "") { throw contractError("no_request_id", "run request needs a request id"); } + const input = offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA) + ? { + schema: MODEL_TEXT_INPUT_V2_SCHEMA, + messages: messages.map(({ role, content }) => ({ + role: role === "agent" ? "assistant" : role, + content, + })), + } + : { + schema: MODEL_TEXT_INPUT_SCHEMA, + prompt: transcriptPrompt(messages), + }; + if (offer.context && Number.isInteger(maxOutputTokens) && maxOutputTokens > 0) { + input.max_output_tokens = maxOutputTokens; + } return { offer_id: offer.offerId, operation: offer.operation, request_id: requestId, - input: offer.inputSchemas?.includes(MODEL_TEXT_INPUT_V2_SCHEMA) - ? { - schema: MODEL_TEXT_INPUT_V2_SCHEMA, - messages: messages.map(({ role, content }) => ({ - role: role === "agent" ? "assistant" : role, - content, - })), - } - : { - schema: MODEL_TEXT_INPUT_SCHEMA, - prompt: transcriptPrompt(messages), - }, + input, }; } diff --git a/capsules/model-provider/src/adapters.rs b/capsules/model-provider/src/adapters.rs index 7dd93bb61..b026cd5a0 100644 --- a/capsules/model-provider/src/adapters.rs +++ b/capsules/model-provider/src/adapters.rs @@ -476,6 +476,7 @@ enum LocalTextBackend { engine: LocalArtifactConfig, model: LocalArtifactConfig, settings: LocalLlamaSettings, + requested_output_tokens: Option, }, } @@ -1155,6 +1156,7 @@ impl AdapterExecutor for LiveAdapterExecutor { engine: engine.clone(), model: model.clone(), settings: settings.clone(), + requested_output_tokens: requested_local_output_tokens(input, settings)?, }, offer, binding, @@ -1298,6 +1300,14 @@ fn validate_text_messages( offer: &ConfiguredOffer, input: &Value, ) -> std::result::Result, AdapterFault> { + if input.get("max_output_tokens").is_some() + && !matches!(offer.adapter, AdapterConfig::LocalLlamaCppText { .. }) + { + return Err(AdapterFault::context( + "model output request is not supported by this offer", + "max_output_tokens is supported only by local llama offers", + )); + } let invalid = || { AdapterFault::context( "model input is invalid", @@ -1327,6 +1337,28 @@ fn validate_text_messages( } } +fn requested_local_output_tokens( + input: &Value, + settings: &LocalLlamaSettings, +) -> std::result::Result, AdapterFault> { + let Some(value) = input.get("max_output_tokens") else { + return Ok(None); + }; + let requested = value.as_u64().ok_or_else(|| { + AdapterFault::context( + "model output request is invalid", + "max_output_tokens must be a positive integer", + ) + })?; + if requested == 0 || requested > u64::from(crate::config::local_output_token_limit(settings)) { + return Err(AdapterFault::context( + "model output request exceeds the profile", + "max_output_tokens exceeds local model limit", + )); + } + Ok(Some(requested as u32)) +} + pub(crate) fn is_local_text_backend_state(value: &Value) -> bool { value .get("schema") @@ -1684,7 +1716,7 @@ async fn run_local_text_worker_inner( ) .then(HostedBackendReport::default); let mut local_socket = None; - let (api_url, api_key, body, private_endpoint) = match &task.backend { + let (api_url, api_key, body, private_endpoint, context_window_tokens) = match &task.backend { LocalTextBackend::OpenRouterDecisions { api_url, api_key, @@ -1714,6 +1746,7 @@ async fn run_local_text_worker_inner( api_key.clone(), text_generation_request_body(&task.offer, model, &task.messages, None), false, + None, ), LocalTextBackend::OpenAiResponses { api_url, @@ -1736,6 +1769,7 @@ async fn run_local_text_worker_inner( })?, ), false, + None, ), LocalTextBackend::LocalLlama { engines, @@ -1743,6 +1777,7 @@ async fn run_local_text_worker_inner( engine, model, settings, + requested_output_tokens, } => { let endpoint = engines .endpoint_with_timeout( @@ -1754,14 +1789,22 @@ async fn run_local_text_worker_inner( ) .await .map_err(map_local_llama_fault)?; - let body = text_generation_request_body( + let mut body = text_generation_request_body( &task.offer, &endpoint.model, &task.messages, Some(endpoint.enable_thinking), ); + body["max_tokens"] = json!(requested_output_tokens + .unwrap_or_else(|| { crate::config::local_output_token_limit(settings) })); local_socket = endpoint.unix_socket; - (endpoint.api_url, None, body, true) + ( + endpoint.api_url, + None, + body, + true, + Some(settings.context_size / settings.parallel), + ) } }; let client = backend_client_with_socket( @@ -1776,6 +1819,52 @@ async fn run_local_text_worker_inner( } else { task.hosted_socket.as_deref() }; + if let Some(context_window_tokens) = context_window_tokens { + let count_url = format!("{api_url}/input_tokens"); + let count_request = client + .post(count_url) + .header("content-type", "application/json") + .json(&body); + let counted = tokio::select! { + _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()), + response = count_request.send() => response.map_err(|err| map_text_reqwest_failure(err, true))?, + }; + if !counted.status().is_success() { + return Err(map_effect_http_status(&counted, None)); + } + let count = tokio::select! { + _ = task.cancel_rx.changed() => return Ok(worker_settlement_unknown_result()), + response = read_bounded_json_response_async(counted) => response?, + }; + let input_tokens = count["input_tokens"] + .as_u64() + .filter(|count| *count > 0) + .ok_or_else(|| { + AdapterFault::malformed( + "model backend returned invalid data", + "missing chat input token count", + ) + })?; + let output_tokens = body["max_tokens"].as_u64().unwrap_or(0); + if input_tokens.saturating_add(output_tokens) >= u64::from(context_window_tokens) { + return Err(AdapterFault::context( + "model context is full", + format!("templated input {input_tokens} plus output {output_tokens} exceeds context {context_window_tokens}"), + )); + } + } + // The count and generation share the one run deadline. + let client = if context_window_tokens.is_some() { + backend_client_with_socket( + remaining_run_timeout(task.deadline_ms)? + .as_millis() + .try_into() + .unwrap_or(u64::MAX), + local_socket.as_deref(), + )? + } else { + client + }; let request = { let effect = if matches!(&task.backend, LocalTextBackend::OpenAiResponses { .. }) { "responses" diff --git a/capsules/model-provider/src/config.rs b/capsules/model-provider/src/config.rs index 701530ffb..29fcff30c 100644 --- a/capsules/model-provider/src/config.rs +++ b/capsules/model-provider/src/config.rs @@ -412,6 +412,10 @@ impl LocalLlamaSettings { MAX_LOCAL_LLAMA_PARALLEL, "local llama parallel", )?; + anyhow::ensure!( + self.context_size / self.parallel >= 128, + "local llama context_size must provide at least 128 tokens per slot" + ); validate_nonzero_bounded(self.threads, MAX_LOCAL_LLAMA_THREADS, "local llama threads")?; validate_nonzero_bounded( self.batch_threads, @@ -579,6 +583,15 @@ impl ConfiguredOffer { output_modalities: self.output_modalities.clone(), stream_output: self.adapter.stream_output(), policy: self.policy.summary(), + context: match &self.adapter { + AdapterConfig::LocalLlamaCppText { settings, .. } => { + Some(crate::contract::LocalContextLimits { + context_window_tokens: settings.context_size / settings.parallel, + max_output_tokens: local_output_token_limit(settings), + }) + } + _ => None, + }, hosted, } } @@ -588,6 +601,7 @@ impl ConfiguredOffer { pub fn execution_summary(&self) -> OfferSummary { let mut summary = self.summary(); summary.input_schemas.clear(); + summary.context = None; summary } @@ -711,6 +725,12 @@ impl ConfiguredOffer { } } +pub(crate) fn local_output_token_limit(settings: &LocalLlamaSettings) -> u32 { + // Keep a conservative answer slice in each server slot. Output bytes have + // their own independent transport bound. + (settings.context_size / settings.parallel / 4).clamp(1, 1_024) +} + pub fn journal_root(base_path: &str, configured: Option<&str>) -> Result { if let Some(configured) = configured { validate_bounded_trimmed(configured, "journal_dir", MAX_JOURNAL_DIR_BYTES)?; @@ -1261,6 +1281,27 @@ mod tests { assert_ne!(responses.execution_binding_hash().unwrap(), legacy_hash); } + #[test] + fn local_context_summary_uses_one_slot_and_rejects_unusable_profiles() { + let mut offer = local_llama_offer(Path::new("/tmp/engine"), Path::new("/tmp/model")); + let original_revision = offer.execution_binding_hash().unwrap(); + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 4_096); + assert_eq!(limits.max_output_tokens, 1_024); + if let AdapterConfig::LocalLlamaCppText { settings, .. } = &mut offer.adapter { + settings.parallel = 16; + assert_eq!(settings.context_size / settings.parallel, 256); + settings.validate().unwrap(); + settings.parallel = 64; + assert!(settings.validate().is_err()); + settings.parallel = 16; + } + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 256); + assert_eq!(limits.max_output_tokens, 64); + assert_ne!(offer.execution_binding_hash().unwrap(), original_revision); + } + #[test] fn execution_binding_hash_changes_only_for_semantic_execution_inputs() { let openai = base_offer(); diff --git a/capsules/model-provider/src/contract.rs b/capsules/model-provider/src/contract.rs index 57ba3e7ed..15e021272 100644 --- a/capsules/model-provider/src/contract.rs +++ b/capsules/model-provider/src/contract.rs @@ -245,9 +245,18 @@ pub struct OfferSummary { pub stream_output: bool, pub policy: OfferPolicySummary, #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub hosted: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LocalContextLimits { + pub context_window_tokens: u32, + pub max_output_tokens: u32, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct HostedOfferDisclosure { diff --git a/capsules/model-provider/src/execution.rs b/capsules/model-provider/src/execution.rs index c5e19d4da..9a2ba234f 100644 --- a/capsules/model-provider/src/execution.rs +++ b/capsules/model-provider/src/execution.rs @@ -3233,6 +3233,180 @@ mod tests { ); } + #[test] + fn hosted_text_rejects_requested_output_limit_before_dispatch() { + let server = start_server(vec![]); + let v1 = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V1_SCHEMA, + "prompt": "hello", + "max_output_tokens": 64, + }); + let v2 = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"hello"}], + "max_output_tokens": 64, + }); + for (offer, input) in [ + (local_text_offer(&server.base_url), v1.clone()), + (local_text_offer(&server.base_url), v2), + (responses_text_offer(&server.base_url), v1), + ] { + let root = temp_root("hosted-output-limit"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let binding = create_binding("request:hosted-output-limit", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert_eq!(terminal["data"]["status"], "failed", "{terminal:#}"); + assert_eq!( + terminal["data"]["terminal"]["error"]["code"], "context_rejected", + "{terminal:#}" + ); + assert_eq!( + terminal["data"]["terminal"]["error"]["message"], + "model output request is not supported by this offer" + ); + provider.shutdown_on_eof(); + } + assert!(server.requests.lock().unwrap().is_empty()); + } + + #[cfg(unix)] + #[test] + fn local_llama_counts_template_before_generation_and_bounds_output() { + let root = temp_root("local-llama-context-budget"); + let (offer, events, root) = local_llama_offer(&root, "healthy"); + let limits = offer.summary().context.unwrap(); + assert_eq!(limits.context_window_tokens, 256); + assert_eq!(limits.max_output_tokens, 64); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + + let accepted_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [ + {"role":"system","content":"Keep the roles."}, + {"role":"user","content":"日本語 🌎"}, + ], + "max_output_tokens": 32, + }); + let accepted_binding = create_binding("request:budget-unicode", &offer, &accepted_input); + let accepted = create_run(&provider, &offer, &accepted_binding, &accepted_input); + let accepted_run = accepted["data"]["run_id"].as_str().unwrap(); + let terminal = + wait_for_terminal(&provider, accepted_run, &access_binding(&accepted_binding)); + assert_eq!(terminal["data"]["status"], "completed", "{terminal:#}"); + let first_events = fake_llama_events(&events); + let count_index = first_events + .iter() + .position(|line| line.starts_with("count:")) + .unwrap(); + let request_index = first_events + .iter() + .position(|line| line == "request:Keep the roles.") + .unwrap(); + assert!(count_index < request_index); + assert!(first_events.iter().any(|line| line == "max_tokens:32")); + let prompt_tokens: u32 = first_events[count_index]["count:".len()..].parse().unwrap(); + assert!(prompt_tokens + 32 < limits.context_window_tokens); + + let long_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"x".repeat(200)}], + "max_output_tokens": 64, + }); + let long_binding = create_binding("request:budget-long", &offer, &long_input); + let long_run = create_run(&provider, &offer, &long_binding, &long_input); + let long_terminal = wait_for_terminal( + &provider, + long_run["data"]["run_id"].as_str().unwrap(), + &access_binding(&long_binding), + ); + assert_eq!( + long_terminal["data"]["terminal"]["error"]["code"], + "context_rejected" + ); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 1 + ); + + let excessive_input = json!({ + "schema": elastos_model_contract::TEXT_INPUT_V2_SCHEMA, + "messages": [{"role":"user","content":"short"}], + "max_output_tokens": 65, + }); + let excessive_binding = + create_binding("request:budget-excessive", &offer, &excessive_input); + let excessive = create_run(&provider, &offer, &excessive_binding, &excessive_input); + assert_eq!( + excessive["data"]["terminal"]["error"]["code"], + "context_rejected" + ); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 1 + ); + provider.shutdown_on_eof(); + } + + #[cfg(unix)] + #[test] + fn local_llama_missing_count_fails_before_generation() { + let root = temp_root("local-llama-missing-count"); + let (offer, events, root) = local_llama_offer(&root, "incomplete_count"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let input = text_input("hello"); + let binding = create_binding("request:missing-count", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert_eq!(terminal["data"]["status"], "failed"); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 0 + ); + provider.shutdown_on_eof(); + } + + #[cfg(unix)] + #[test] + fn local_llama_stop_interrupts_stalled_count_body() { + let root = temp_root("local-llama-stalled-count"); + let (offer, events, root) = local_llama_offer(&root, "stalled_count"); + let mut provider = ProviderCoordinatorHandle::start(); + init_provider(&provider, &root, vec![offer.clone()]); + let input = text_input("hello"); + let binding = create_binding("request:stalled-count", &offer, &input); + let created = create_run(&provider, &offer, &binding, &input); + let run_id = created["data"]["run_id"].as_str().unwrap(); + wait_for_fake_llama_event(&events, "count_body_started"); + let started = Instant::now(); + let _ = cancel_run(&provider, run_id, &access_binding(&binding)); + let terminal = wait_for_terminal(&provider, run_id, &access_binding(&binding)); + assert!(started.elapsed() < Duration::from_secs(2)); + assert_eq!(terminal["data"]["status"], "settlement_unknown"); + assert_eq!( + fake_llama_events(&events) + .iter() + .filter(|line| line.starts_with("request:")) + .count(), + 0 + ); + provider.shutdown_on_eof(); + } + #[cfg(unix)] #[test] fn local_llama_streams_concurrent_runs_reuses_child_and_guards_refresh() { diff --git a/capsules/model-provider/src/journal.rs b/capsules/model-provider/src/journal.rs index 3fb53f0f8..dfbf1d1df 100644 --- a/capsules/model-provider/src/journal.rs +++ b/capsules/model-provider/src/journal.rs @@ -517,6 +517,18 @@ fn validate_stored_offer(path: &Path, offer: &OfferSummary) -> Result<(), Provid )?; validate_modalities(path, "input", &offer.input_modalities)?; validate_modalities(path, "output", &offer.output_modalities)?; + if let Some(context) = &offer.context { + if context.context_window_tokens < 128 + || context.context_window_tokens > crate::config::MAX_LOCAL_LLAMA_CONTEXT_SIZE + || context.max_output_tokens == 0 + || context.max_output_tokens >= context.context_window_tokens + { + return Err(ProviderFault::corrupt_journal(format!( + "model run journal context limit is invalid at {}", + path.display() + ))); + } + } if let Some(hosted) = offer.hosted.as_ref() { if hosted.placement != HOSTED_PLACEMENT || hosted.selection_mode != HOSTED_SELECTION_PINNED diff --git a/capsules/model-provider/src/local_llama.rs b/capsules/model-provider/src/local_llama.rs index 38a89b011..e43df4023 100644 --- a/capsules/model-provider/src/local_llama.rs +++ b/capsules/model-provider/src/local_llama.rs @@ -367,6 +367,7 @@ fn engine_arguments( args.extend([ "--ctx-size".into(), settings.context_size.to_string().into(), + "--no-context-shift".into(), "--parallel".into(), settings.parallel.to_string().into(), "--threads".into(), @@ -952,6 +953,7 @@ mod tests { "11434", "--ctx-size", "256", + "--no-context-shift", "--parallel", "2", "--threads", diff --git a/capsules/model-provider/src/state.rs b/capsules/model-provider/src/state.rs index 0a1e6e3b9..83823a1c7 100644 --- a/capsules/model-provider/src/state.rs +++ b/capsules/model-provider/src/state.rs @@ -907,6 +907,7 @@ impl ModelProviderState { let current_summary = offer.execution_summary(); let mut recorded_summary = run.offer.clone(); recorded_summary.input_schemas.clear(); + recorded_summary.context = None; let current_hash = offer.execution_binding_hash().map_err(|err| { ProviderFault::internal(format!( "failed to derive model execution binding hash: {err}" diff --git a/capsules/model-provider/src/test_support.rs b/capsules/model-provider/src/test_support.rs index f60880234..9cca2c473 100644 --- a/capsules/model-provider/src/test_support.rs +++ b/capsules/model-provider/src/test_support.rs @@ -163,7 +163,7 @@ class Handler(http.server.BaseHTTPRequestHandler): self.wfile.write(json.dumps(body).encode('utf-8')) def do_POST(self): - if self.path != '/v1/chat/completions': + if self.path not in ('/v1/chat/completions', '/v1/chat/completions/input_tokens'): self.send_error(404) return length = int(self.headers.get('Content-Length', '0')) @@ -172,10 +172,42 @@ class Handler(http.server.BaseHTTPRequestHandler): if body.get('model') != alias or body.get('chat_template_kwargs') != {'enable_thinking': False}: self.send_error(400) return + if self.path == '/v1/chat/completions/input_tokens': + # A fixture tokenizer charges UTF-8 content and per-message template + # markers. The pinned engine's endpoint owns exact production counts. + count = sum(len(message['content'].encode('utf-8')) + 8 for message in body['messages']) + record('count:' + str(count)) + response = {'object': 'response.input_tokens', 'input_tokens': count} + if mode == 'incomplete_count': + del response['input_tokens'] + if mode == 'stalled_count': + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', '100') + self.end_headers() + self.wfile.write(b'{') + self.wfile.flush() + record('count_body_started') + while True: + try: + self.wfile.write(b' ') + self.wfile.flush() + except (BrokenPipeError, ConnectionResetError): + record('count_disconnected') + return + time.sleep(0.025) + data = json.dumps(response).encode('utf-8') + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + return active = model_path.with_suffix('.active') if mode == 'active_after_disconnect' and prompt == 'stall': active.touch() record('request:' + prompt) + record('max_tokens:' + str(body.get('max_tokens'))) if mode == 'crash_once' and prompt == 'crash': marker = model_path.with_suffix('.crashed') if not marker.exists(): diff --git a/docs/MODEL_PROVIDER.md b/docs/MODEL_PROVIDER.md index f07c114f0..88888e239 100644 --- a/docs/MODEL_PROVIDER.md +++ b/docs/MODEL_PROVIDER.md @@ -141,6 +141,17 @@ The provider binds these facts before dispatch: Secrets, raw provider credentials, and private endpoint details do not belong in the session journal or App-visible result. +For a local llama offer, `offers_list` includes `context.context_window_tokens` +for one engine slot and `context.max_output_tokens` for the selected profile. +Assistant packs against these values. For local llama offers only, a text/v1 or +text/v2 input can include `max_output_tokens`; Runtime binds that value in the +input hash. Hosted offers reject this field with `context_rejected`. An omitted +value uses the local profile limit. Before generation, the provider asks the pinned +engine to count the exact chat request with its template and special tokens. +It sends generation only when the counted input and requested answer fit the +slot. The offer's input and output byte policies remain separate transport +bounds. + ## Stream lifecycle A model request follows one observable lifecycle: diff --git a/elastos/crates/elastos-model-contract/src/lib.rs b/elastos/crates/elastos-model-contract/src/lib.rs index e6d218106..4b7299491 100644 --- a/elastos/crates/elastos-model-contract/src/lib.rs +++ b/elastos/crates/elastos-model-contract/src/lib.rs @@ -163,7 +163,13 @@ pub fn model_input_hash(input: &Value) -> ContractResult { pub fn validate_text_input_v2(input: &Value) -> ContractResult<&[Value]> { let invalid = || ContractError::new("invalid text/v2 conversation"); let object = input.as_object().ok_or_else(invalid)?; - if object.len() != 2 || input["schema"] != TEXT_INPUT_V2_SCHEMA { + if (object.len() != 2 && object.len() != 3) + || input["schema"] != TEXT_INPUT_V2_SCHEMA + || (object.len() == 3 + && input["max_output_tokens"] + .as_u64() + .is_none_or(|count| count == 0 || count > u32::MAX as u64)) + { return Err(invalid()); } let messages = input["messages"].as_array().ok_or_else(invalid)?; @@ -287,6 +293,19 @@ mod tests { json!(vec![user.clone(); MAX_TEXT_MESSAGES + 1]), ]; assert!(validate_text_input_v2(&extra_top).is_err()); + let bounded = + json!({"schema":TEXT_INPUT_V2_SCHEMA,"messages":[user.clone()],"max_output_tokens":32}); + assert!(validate_text_input_v2(&bounded).is_ok()); + for invalid_limit in [ + json!(0), + json!(-1), + json!("32"), + json!(u64::from(u32::MAX) + 1), + ] { + let mut invalid = bounded.clone(); + invalid["max_output_tokens"] = invalid_limit; + assert!(validate_text_input_v2(&invalid).is_err()); + } assert!( validate_text_input_v2(&json!({"schema":TEXT_INPUT_V1_SCHEMA,"messages":[user]})) .is_err() diff --git a/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs b/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs index 478838e90..5538f5c2c 100644 --- a/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs +++ b/elastos/crates/elastos-runtime/src/provider/local_model_broker.rs @@ -102,7 +102,9 @@ async fn forward( let route = &request[..line_end]; if !matches!( route, - b"GET /v1/models HTTP/1.1" | b"POST /v1/chat/completions HTTP/1.1" + b"GET /v1/models HTTP/1.1" + | b"POST /v1/chat/completions HTTP/1.1" + | b"POST /v1/chat/completions/input_tokens HTTP/1.1" ) { return Err(io::Error::new( io::ErrorKind::PermissionDenied, @@ -176,7 +178,11 @@ fn request_body_length(headers: &[u8], route: &[u8]) -> io::Result { let length = length.unwrap_or(0); if length > MAX_REQUEST_BODY_BYTES || (route == b"GET /v1/models HTTP/1.1" && length != 0) - || (route == b"POST /v1/chat/completions HTTP/1.1" && length == 0) + || (matches!( + route, + b"POST /v1/chat/completions HTTP/1.1" + | b"POST /v1/chat/completions/input_tokens HTTP/1.1" + ) && length == 0) { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -310,7 +316,9 @@ mod tests { let engine = dir.path().join("engine.sock"); let mut child = tokio::process::Command::new("/usr/bin/python3") .arg("-c") - .arg(r"import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(1); c,_=s.accept(); c.recv(4096); c.sendall(b'HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok'); c.close()") + .arg(r"import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.listen(2) +for _ in range(2): + c,_=s.accept(); request=c.recv(4096); body=b'ok' if request.startswith(b'GET /v1/models ') else b'counted' if request.startswith(b'POST /v1/chat/completions/input_tokens ') else b'wrong'; c.sendall(b'HTTP/1.1 200 OK\r\nContent-Length: '+str(len(body)).encode()+b'\r\nConnection: close\r\n\r\n'+body); c.close()") .arg(&engine) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -351,6 +359,20 @@ mod tests { .unwrap() .unwrap(); assert!(response.ends_with(b"\r\n\r\nok")); + let mut count = UnixStream::connect(&broker).await.unwrap(); + count + .write_all( + b"POST /v1/chat/completions/input_tokens HTTP/1.1\r\nContent-Length: 2\r\n\r\n{}", + ) + .await + .unwrap(); + count.shutdown().await.unwrap(); + let mut counted = Vec::new(); + tokio::time::timeout(Duration::from_secs(2), count.read_to_end(&mut counted)) + .await + .unwrap() + .unwrap(); + assert!(counted.ends_with(b"\r\n\r\ncounted")); assert!(child.wait().await.unwrap().success()); task.abort(); } @@ -375,15 +397,19 @@ mod tests { #[test] fn one_request_body_is_bounded_and_chunked_or_duplicate_lengths_are_refused() { - let route = b"POST /v1/chat/completions HTTP/1.1"; - assert_eq!( - request_body_length( - b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 2\r\n\r\n", - route - ) - .unwrap(), - 2 - ); + for route in [ + b"POST /v1/chat/completions HTTP/1.1".as_slice(), + b"POST /v1/chat/completions/input_tokens HTTP/1.1".as_slice(), + ] { + let header = format!( + "{}\r\nContent-Length: 2\r\n\r\n", + std::str::from_utf8(route).unwrap() + ); + assert_eq!(request_body_length(header.as_bytes(), route).unwrap(), 2); + assert!(request_body_length(b"Content-Length: 0\r\n\r\n", route).is_err()); + assert!(request_body_length(b"Content-Length: 4194305\r\n\r\n", route).is_err()); + } + let route = b"POST /v1/chat/completions/input_tokens HTTP/1.1"; for header in [ b"POST /v1/chat/completions HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n".as_slice(), b"POST /v1/chat/completions HTTP/1.1\r\nContent-Length: 2\r\nContent-Length: 3\r\n\r\n", diff --git a/scripts/home-agent-cancellation.test.mjs b/scripts/home-agent-cancellation.test.mjs index da5a372be..ec5a82272 100644 --- a/scripts/home-agent-cancellation.test.mjs +++ b/scripts/home-agent-cancellation.test.mjs @@ -24,6 +24,7 @@ const workspace = await import("../capsules/assistant/browser/agent-workspace.js const sessions = await import("../capsules/assistant/browser/agent-sessions.js"); const defer = () => { let resolve, reject; const promise = new Promise((r, e) => { resolve = r; reject = e; }); return { promise, resolve, reject }; }; const terminal = (kind) => ({ events: [{ sequence: 1, kind, terminal: true, data: kind === "output" ? { schema: "elastos.model.output.text/v1", text: "Done" } : {} }], next_cursor: 1, has_more: false }); +const fixtureContext = { context_window_tokens: 4096, max_output_tokens: 1024 }; test("content intent survives missing catalog and overlapping refresh; hosted replacement is deliberate", async t => { t.after(() => live.selectLiveOffer("")); @@ -307,7 +308,7 @@ async function fixture(cancelReject = false, saved = null) { const body = JSON.parse(init.body); calls.push({ op, body }); let data; - if (op === "offers_list") data = { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }] }; + if (op === "offers_list") data = { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }] }; else if (op === "runs_create") data = { run_id: "run-fixture", sequence_cursor: 0 }; else if (op === "runs_get") data = { run_id: "run-fixture", status: "running" }; else if (op === "runs_events") data = await events.promise; @@ -375,7 +376,7 @@ test("an immediately failed create settles without polling events", async () => const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-full", sequence_cursor: 0, status: "failed", @@ -397,7 +398,7 @@ test("a retained unknown create settles from the create view without polling", a const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-pruned", sequence_cursor: 0, status: "settlement_unknown", @@ -447,7 +448,7 @@ test("an attached poll at cursor 7 settles from a retained terminal event", asyn const body = init?.body ? JSON.parse(init.body) : null; calls.push({ op, body }); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-attached", sequence_cursor: 7, status: "running", @@ -477,7 +478,7 @@ test("a completed pruned journal states that live output was not retained", asyn const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-completed-pruned", sequence_cursor: 0, status: "completed", @@ -504,7 +505,7 @@ test("an attached poll of a completed pruned journal states that output was not const body = init?.body ? JSON.parse(init.body) : null; calls.push({ op, body }); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-completed-events", sequence_cursor: 2, status: "running", @@ -609,7 +610,7 @@ for (const mode of ["lost_response", "invalid_json", "missing_run_id", "invalid_ globalThis.fetch = async (url, init) => { const op = new URL(url).pathname.split("/").pop(); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; assert.equal(op, "runs_create"); const body = JSON.parse(init.body); @@ -794,7 +795,7 @@ async function runUnretainedLiveTurn(t, { create, eventsPages = [] }) { if (op === "offers_list") { offerLists += 1; return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; } if (op === "runs_create") return { ok: true, json: async () => create }; @@ -889,7 +890,7 @@ test("immediate failed create shows the provider result and restores it after sa const op = new URL(url).pathname.split("/").pop(); calls.push(op); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op === "runs_create") return { ok: true, json: async () => ({ run_id: "run-overflow", sequence_cursor: 2, status: "failed", @@ -1115,7 +1116,7 @@ test("actual composer preserves draft for missing runs and unknown acceptance un calls.push({ op, body: JSON.parse(init.body) }); if (op === "runs_get") return { ok: false, status: 404, json: async () => ({ code: "run_not_found", message: "model run not found" }) }; return { ok: true, json: async () => op === "offers_list" - ? { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }] } + ? { offers: [{ id: "fixture", title: "Fixture", operation: "text", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }] } : op === "runs_create" ? { run_id: "new-draft-run", sequence_cursor: 0 } : terminal("settlement_unknown") }; }; const draft = { text: "new unsent prompt", parts: [{ id: "attachment", kind: "file", name: "notes.txt", text: "reference" }] }; @@ -1182,7 +1183,7 @@ test("only a matching fresh invocation refusal settles a failed create", async ( globalThis.fetch = async (url, init) => { const op = new URL(url).pathname.split("/").pop(); if (op === "offers_list") return { ok: true, json: async () => ({ offers: [ - { id: offerId, title: "Fixture", operation: "text.generate", input_modalities: ["text/plain"], output_modalities: ["text/plain"] }, + { id: offerId, title: "Fixture", operation: "text.generate", input_modalities: ["text/plain"], output_modalities: ["text/plain"], context: fixtureContext }, ] }) }; if (op !== "runs_create") throw new Error(`unexpected ${op}`); const body = JSON.parse(init.body); calls.push(body); diff --git a/scripts/home-agent-shell-smoke.mjs b/scripts/home-agent-shell-smoke.mjs index b75b22961..1f5a56aaa 100644 --- a/scripts/home-agent-shell-smoke.mjs +++ b/scripts/home-agent-shell-smoke.mjs @@ -68,7 +68,7 @@ assert.ok( "Assistant Chat and Studio decode events through the typed contract module", ); assert.ok(agentLive.includes("after_sequence: afterSequence"), "runs_events is polled by after_sequence"); -assert.ok(agentLive.includes("textRunCreateBody({ offer, messages, requestId: createRequestId })")); +assert.ok(agentLive.includes("maxOutputTokens: manifest.outputReserve")); assert.ok(agentLive.includes('modelRunCall("runs_cancel", { run_id: runId, request_id: newRequestId() })')); assert.ok(!agentLive.includes("agent-run-cursor"), "the URUX cursor helper is gone"); @@ -236,6 +236,7 @@ assert.ok(!/Planning weekend|calm weekend/.test(read("capsules/assistant/browser /* ---- the pure contract module -------------------------------------------- */ const contract = await import(new URL("model-contract.js", capsuleDir)); +const context = await import(new URL("agent-context.js", capsuleDir)); const homeMessageContract = await import( new URL("capsules/home-gui/browser/home-agent-message-contract.js", root) ); @@ -356,7 +357,9 @@ assert.deepEqual(rows, [ label: "Local chat", detail: "This Home · local", streamOutput: true, + routeKind: "local", inputSchemas: [], + context: null, selectionFacts: localFacts, }, ]); @@ -502,6 +505,42 @@ assert.deepEqual(body, { request_id: "req-1", input: { schema: "elastos.model.input.text/v1", prompt: contract.transcriptPrompt(messages) }, }); +const boundedLocal = contract.textOfferRows([{ ...textOffers[0], context: { + context_window_tokens: 256, max_output_tokens: 64, +} }])[0]; +const effective = context.capabilitiesForOffer(boundedLocal); +assert.deepEqual(effective.context, { + ...context.TEXT_CONTRACT_CAPABILITIES.context, + maxInputTokens: 256, + maxOutputTokens: 64, +}); +assert.equal(context.makeBudget(effective.context, 32).outputReserve, 32); +assert.ok(context.makeBudget(effective.context, 32).hardEstimatedInputLimit < 256 - 32); +const longHistory = Array.from({ length: 25 }, (_, i) => ({ + id: `turn-${i}`, role: i % 2 ? "assistant" : "user", content: `turn ${i} ${"x".repeat(400)}`, +})); +const packed = context.compileContext({ + history: longHistory, + currentInput: { id: "current", role: "user", content: "Continue" }, + constraints: [{ id: "system", kind: "system_policy", role: "system", content: "Be brief.", mustInclude: true }], + budget: context.makeBudget(effective.context, 32), + capabilities: effective, +}); +assert.ok(packed.omitted.length > 0); +assert.ok(packed.manifest.estimatedInputTokens <= packed.budget.hardEstimatedInputLimit); +assert.equal(packed.manifest.outputReserve, 32); +for (const missing of [null, {}, { context_window_tokens: 256 }, + { context_window_tokens: 256, max_output_tokens: 256 }]) { + assert.throws(() => context.capabilitiesForOffer({ ...boundedLocal, context: missing }), + { code: "context_overflow" }); +} +assert.deepEqual(contract.textRunCreateBody({ + offer: boundedLocal, messages, requestId: "bounded", maxOutputTokens: 32, +}).input, { + schema: contract.MODEL_TEXT_INPUT_SCHEMA, + prompt: contract.transcriptPrompt(messages), + max_output_tokens: 32, +}); // Follow the serialized offer response through selection into the run request. const conversation = [ { role: "system", content: " Keep spacing. " },