//! LF-delimited JSON-RPC over standard input and output (`SMH-SPEC-SPEC0001`, //! Interface modes). //! //! Standard output carries protocol records only. //! Request IDs are preserved on every response. //! Durable control remains available while one executor blocks on provider work. #![deny(unsafe_code)] use smith::error::Result; use smith::id::EntryId; use smith_core::control::{ AccessMode, DurableControl, DurableExecutor, ExecutionPhase, WorkReceipt, }; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; use std::sync::mpsc::{self, Receiver, SyncSender}; /// Commands held for the executor thread. const COMMAND_QUEUE: usize = 64; /// Response lines held before the transport drains them. const RESPONSE_QUEUE: usize = 256; /// RPC state with independent durable control and blocking execution planes. pub struct RpcState { control: Arc, commands: SyncSender, responses: Receiver, pending: Arc, admitted: Vec, latest_admission: Option, waits: SyncSender, } impl RpcState { /// Start an RPC session over independently-owned durable contracts. #[must_use] pub fn new(control: Arc, executor: impl DurableExecutor + 'static) -> Self { Self::with_executor(control, Some(Box::new(executor))) } /// Start an inspection-only RPC session. #[must_use] pub fn inspection(control: Arc) -> Self { Self::with_executor(control, None) } fn with_executor( control: Arc, executor: Option>, ) -> Self { let (command_tx, command_rx) = mpsc::sync_channel(COMMAND_QUEUE); let (response_tx, response_rx) = mpsc::sync_channel(RESPONSE_QUEUE); let pending = Arc::new(AtomicUsize::new(0)); let (wait_tx, wait_rx) = mpsc::sync_channel(COMMAND_QUEUE); let worker = Worker { control: Arc::clone(&control), executor, responses: response_tx, pending: Arc::clone(&pending), waits: wait_rx, pending_waits: Vec::with_capacity(COMMAND_QUEUE), }; std::thread::spawn(move || worker.run(command_rx)); Self { control, commands: command_tx, responses: response_rx, pending, admitted: Vec::with_capacity(COMMAND_QUEUE), latest_admission: None, waits: wait_tx, } } /// Remove completed asynchronous response lines without blocking. #[must_use] pub fn drain_lines(&self) -> Vec { self.responses .try_iter() .flat_map(|response| line_or_empty(&response)) .collect() } /// Block up to `timeout` for the next response line. #[must_use] pub fn recv_line(&self, timeout: std::time::Duration) -> Option { self.responses.recv_timeout(timeout).ok()?.to_line().ok() } /// Whether no accepted prompt is executing. #[must_use] pub fn is_settled(&self) -> bool { self.pending.load(SeqCst) == 0 } } #[derive(Clone, Debug, PartialEq, Eq)] struct RpcRequest { id: serde_json::Value, method: String, params: serde_json::Value, } #[derive(Clone, Debug, PartialEq, Eq)] struct RpcResponse { id: serde_json::Value, record: serde_json::Value, } impl RpcResponse { fn to_line(&self) -> Result { let mut value = serde_json::json!({"id": self.id}); merge(&mut value, &self.record); let mut line = serde_json::to_string(&value).map_err(|error| smith::error::SmithError::Session { code: smith::error::SessionCode::RpcEncode, message: error.to_string(), })?; line.push('\n'); Ok(line) } } struct ReservedExecution { request: RpcRequest, admission: Receiver>, } struct RootWait { request: RpcRequest, root: EntryId, responded: Arc, } struct Worker { control: Arc, executor: Option>, responses: SyncSender, pending: Arc, waits: Receiver, pending_waits: Vec, } impl Worker { #[expect( clippy::needless_pass_by_value, reason = "worker owns its receiver for the lifetime of its dedicated thread" )] fn run(mut self, commands: Receiver) { while let Ok(ReservedExecution { request, admission }) = commands.recv() { if let Ok(Some(id)) = admission.recv() { self.execute(&request, id); self.pending.fetch_sub(1, SeqCst); self.respond_settled_waits(id); } } } fn respond_settled_waits(&mut self, root: EntryId) { while let Ok(wait) = self.waits.try_recv() { if wait.responded.load(SeqCst) { continue; } if self.pending_waits.len() == COMMAND_QUEUE { if !wait.responded.swap(true, SeqCst) { self.send(error(&wait.request, "BUSY", "wait queue is full")); } } else { self.pending_waits.push(wait); } } let mut index = 0; while index < self.pending_waits.len() { if self.pending_waits[index].root == root || self.pending_waits[index].responded.load(SeqCst) { let wait = self.pending_waits.swap_remove(index); if !wait.responded.swap(true, SeqCst) { self.send(root_wait_response( self.control.as_ref(), &wait.request, root, )); } } else { index += 1; } } } fn execute(&mut self, request: &RpcRequest, id: EntryId) { let Some(executor) = self.executor.as_mut() else { self.send(RpcResponse { id: request.id.clone(), record: serde_json::json!({"event": "end", "error": {"code": "STATE", "message": "inspection mode has no executor"}}), }); return; }; let responses = &self.responses; let mut observe = |event: &smith::stream::StreamEvent| { if let smith::stream::StreamEvent::TextDelta { delta, .. } = event { let _send_result = responses.send(result( request, serde_json::json!({"event": "delta", "text": delta}), )); } }; match executor.execute(id, &mut observe) { Ok(outcome) => { self.send(result( request, serde_json::json!({ "event": "end", "text": outcome.text, "reason": format!("{:?}", outcome.reason), "cost": outcome.cost.map(|cost| serde_json::json!({ "input_tokens": cost.input_tokens, "output_tokens": cost.output_tokens, })), "cost_complete": outcome.cost_complete, }), )); } Err(error) => self.send(RpcResponse { id: request.id.clone(), record: serde_json::json!({ "event": "end", "error": {"code": "TURN_FAILED", "message": error.to_string()}, }), }), } } fn send(&self, response: RpcResponse) { let _send_result = self.responses.send(response); } } /// Handle one input line, returning immediate records. #[must_use] pub fn handle_line(state: &mut RpcState, line: &str) -> Vec { let trimmed = line.trim(); if trimmed.is_empty() { return Vec::new(); } let parsed: serde_json::Value = match serde_json::from_str(trimmed) { Ok(value) => value, Err(error) => { return line_or_empty(&RpcResponse { id: serde_json::Value::Null, record: serde_json::json!({"error": {"code": "PARSE", "message": error.to_string()}}), }); } }; let request = RpcRequest { id: parsed.get("id").cloned().unwrap_or(serde_json::Value::Null), method: parsed .get("method") .and_then(serde_json::Value::as_str) .unwrap_or_default() .to_string(), params: parsed .get("params") .cloned() .unwrap_or_else(|| serde_json::json!({})), }; dispatch(state, &request) .into_iter() .flat_map(|response| line_or_empty(&response)) .collect() } fn dispatch(state: &mut RpcState, request: &RpcRequest) -> Vec { match request.method.as_str() { "ping" => vec![result(request, serde_json::json!({"pong": true}))], "prompt" => start_prompt(state, request), "abort" => abort(state, request), "inspect" => inspect(state, request), "status" => vec![status(state, request)], "wait" => wait(state, request), other => vec![error( request, "METHOD_UNKNOWN", &format!("unknown method {other}"), )], } } fn start_prompt(state: &mut RpcState, request: &RpcRequest) -> Vec { let input = request .params .get("input") .and_then(serde_json::Value::as_str) .unwrap_or_default(); let key = request .params .get("key") .and_then(serde_json::Value::as_str); let delivery = match request .params .get("delivery") .and_then(serde_json::Value::as_str) { None | Some("AfterTurn") => smith_core::agent::Delivery::AfterTurn, Some("BetweenRequests") => smith_core::agent::Delivery::BetweenRequests, Some(_) => { return vec![error( request, "PARAMS", "delivery must be BetweenRequests or AfterTurn", )]; } }; let notification = match state.control.execution_phase() { Ok(Some((root, ExecutionPhase::Notifying))) => Some(root), Ok(_) => None, Err(fault) => return vec![error(request, "STATE", &fault.to_string())], }; if !state.is_settled() && notification.is_none() && key.is_none() { return vec![error(request, "BUSY", "a prompt is already running")]; } // Reserve bounded executor ownership BEFORE durable acceptance. A rollback // sends None, never an executable identity or a fabricated accepted receipt. state.admitted.retain(|id| { !matches!(state.control.inspect(*id), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_))) }); if state.pending.load(SeqCst) >= COMMAND_QUEUE || state.admitted.len() >= COMMAND_QUEUE { return vec![error(request, "BUSY", "executor queue is full")]; } let (accepted, admission) = mpsc::sync_channel(1); if state .commands .try_send(ReservedExecution { request: request.clone(), admission, }) .is_err() { return vec![error(request, "STATE", "executor unavailable")]; } let admission = match notification { Some(root) => state.control.admit_notification(root, input, key, delivery), None => state.control.admit(input, key), }; let admission = match admission { Ok(admission) => admission, Err(control_error) => { let _rollback = accepted.send(None); return vec![error( request, if state.is_settled() { "STATE" } else { "BUSY" }, &control_error.to_string(), )]; } }; let receipt = receipt(&admission.work); if !admission.is_new { let _rollback = accepted.send(None); return vec![result( request, serde_json::json!({"event": "start", "duplicate": true, "receipt": receipt}), )]; } state.admitted.push(admission.work.receipt.submission_id); state.latest_admission = Some(admission.work.receipt.submission_id); state.pending.fetch_add(1, SeqCst); if accepted .send(Some(admission.work.receipt.submission_id)) .is_err() { state.pending.fetch_sub(1, SeqCst); return vec![error( request, "STATE", "executor unavailable; admitted work remains queued", )]; } vec![result( request, serde_json::json!({"event": "start", "receipt": receipt}), )] } fn abort(state: &RpcState, request: &RpcRequest) -> Vec { let id = match request.params.get("id") { Some(value) => match stable_id(value) { Ok(id) => Some(id), Err(response) => return vec![error(request, "PARAMS", response)], }, None => None, }; match state.control.abort(id) { Ok(id) => vec![result( request, serde_json::json!({"aborting": true, "submission_id": id.to_string()}), )], Err(control_error) => vec![error(request, "STATE", &control_error.to_string())], } } fn inspect(state: &RpcState, request: &RpcRequest) -> Vec { let Some(value) = request.params.get("id") else { return vec![error(request, "PARAMS", "id is required")]; }; let id = match stable_id(value) { Ok(id) => id, Err(message) => return vec![error(request, "PARAMS", message)], }; match state.control.inspect(id) { Ok(Some(work)) => vec![result( request, serde_json::json!({"receipt": receipt(&work)}), )], Ok(None) => vec![result(request, serde_json::json!({"receipt": null}))], Err(control_error) => vec![error(request, "STATE", &control_error.to_string())], } } fn stable_id(value: &serde_json::Value) -> std::result::Result { let Some(value) = value.as_str() else { return Err("id is not a stable entry identity"); }; serde_json::from_value(serde_json::Value::String(value.to_owned())) .map_err(|_| "id is not a stable entry identity") } fn status(state: &RpcState, request: &RpcRequest) -> RpcResponse { let phase = match state.control.execution_phase() { Ok(phase) => phase, Err(fault) => return error(request, "STATE", &fault.to_string()), }; let mut settled = state.is_settled(); let mut active = phase.map(|(id, _)| id); if settled && let Some(id) = state.latest_admission { match state.control.inspect(id) { Ok(Some(work)) => { settled = matches!(work.status, smith_core::store::WorkStatus::Terminal(_)); if work.status == smith_core::store::WorkStatus::Running { active = Some(id); } } Ok(None) => settled = false, Err(fault) => return error(request, "STATE", &fault.to_string()), } } result( request, serde_json::json!({ "settled": settled, "access_mode": access_mode(state.control.access_mode()), "active_submission_id": active.map(|id| id.to_string()), }), ) } fn wait(state: &RpcState, request: &RpcRequest) -> Vec { let explicit = match request.params.get("id") { Some(value) => match stable_id(value) { Ok(id) => Some(id), Err(message) => return vec![error(request, "PARAMS", message)], }, None => None, }; let phase = match state.control.execution_phase() { Ok(phase) => phase, Err(fault) => return vec![error(request, "STATE", &fault.to_string())], }; // Notification ownership wins until cleanup ends; otherwise bind the oldest // nonterminal admission, including the executor registration handoff. let root = explicit .or_else(|| phase.map(|(id, _)| id)) .or_else(|| state.admitted.iter().copied().find(|id| { !matches!(state.control.inspect(*id), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_))) })) .or(state.latest_admission); let Some(root) = root else { return vec![result(request, serde_json::json!({"settled": true}))]; }; if state.is_settled() || matches!(state.control.inspect(root), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_))) { return vec![root_wait_response(state.control.as_ref(), request, root)]; } let responded = Arc::new(AtomicBool::new(false)); if state .waits .try_send(RootWait { request: request.clone(), root, responded: Arc::clone(&responded), }) .is_err() { return vec![error(request, "STATE", "wait queue unavailable")]; } // Close the enqueue-versus-cleanup race: either worker sees this waiter, // or its receipt is terminal now. The response lease prevents duplication. if let Ok(Some(work)) = state.control.inspect(root) && matches!(work.status, smith_core::store::WorkStatus::Terminal(_)) && !responded.swap(true, SeqCst) { return vec![result(request, serde_json::json!({"settled": true}))]; } Vec::new() } fn root_wait_response( control: &dyn DurableControl, request: &RpcRequest, root: EntryId, ) -> RpcResponse { match control.inspect(root) { Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_)) => { result(request, serde_json::json!({"settled": true})) } Ok(_) => error( request, "STATE", "root cleanup finished without durable terminal acknowledgement", ), Err(fault) => error(request, "STATE", &fault.to_string()), } } fn receipt(work: &WorkReceipt) -> serde_json::Value { serde_json::json!({ "session_id": work.receipt.session_id.to_string(), "submission_id": work.receipt.submission_id.to_string(), "destination": work.receipt.destination.map(|id| id.to_string()), "status": format!("{:?}", work.status), "cancel_requested": work.cancel_requested, }) } const fn access_mode(mode: AccessMode) -> &'static str { match mode { AccessMode::ReadOnly => "read_only", AccessMode::Writer => "writer", } } fn merge(target: &mut serde_json::Value, patch: &serde_json::Value) { if let (Some(target), Some(patch)) = (target.as_object_mut(), patch.as_object()) { target.extend( patch .iter() .map(|(key, value)| (key.clone(), value.clone())), ); } } fn line_or_empty(response: &RpcResponse) -> Vec { response .to_line() .map_or_else(|_| Vec::new(), |line| vec![line]) } fn result(request: &RpcRequest, record: serde_json::Value) -> RpcResponse { RpcResponse { id: request.id.clone(), record, } } fn error(request: &RpcRequest, code: &str, message: &str) -> RpcResponse { RpcResponse { id: request.id.clone(), record: serde_json::json!({"error": {"code": code, "message": message}}), } } #[cfg(test)] mod tests { use super::{RpcState, handle_line}; use smith::error::{Result, SessionCode, SmithError}; use smith::id::{EntryId, SessionId}; use smith_core::control::{ AccessMode, Admission, DurableControl, DurableExecutor, WorkReceipt, }; use smith_core::store::{AdmissionReceipt, WorkStatus}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; struct Control { work: WorkReceipt, admissions: AtomicUsize, aborts: AtomicUsize, } impl DurableControl for Control { fn access_mode(&self) -> AccessMode { AccessMode::Writer } fn admit(&self, _: &str, key: Option<&str>) -> Result { let count = self.admissions.fetch_add(1, Ordering::SeqCst); Ok(Admission { work: self.work, is_new: count == 0 || key.is_none(), }) } fn inspect(&self, id: EntryId) -> Result> { Ok((id == self.work.receipt.submission_id).then_some(self.work)) } fn abort(&self, id: Option) -> Result { let id = id.unwrap_or(self.work.receipt.submission_id); if id != self.work.receipt.submission_id { return Err(SmithError::Session { code: SessionCode::SessionCorrupt, message: "stale submission".into(), }); } self.aborts.fetch_add(1, Ordering::SeqCst); Ok(id) } } struct FailingExecutor; impl DurableExecutor for FailingExecutor { fn execute( &mut self, _id: EntryId, _observe: &mut dyn FnMut(&smith::stream::StreamEvent), ) -> Result { Err(SmithError::Session { code: SessionCode::SessionCorrupt, message: "fixture execution failed".into(), }) } } struct BlockingExecutor { started: SyncSender, release: Receiver<()>, executions: Arc, } impl DurableExecutor for BlockingExecutor { fn execute( &mut self, id: EntryId, _observe: &mut dyn FnMut(&smith::stream::StreamEvent), ) -> Result { self.executions.fetch_add(1, Ordering::SeqCst); self.started.send(id).map_err(|error| SmithError::Session { code: SessionCode::SessionCorrupt, message: error.to_string(), })?; self.release.recv().map_err(|error| SmithError::Session { code: SessionCode::SessionCorrupt, message: error.to_string(), })?; Err(SmithError::Session { code: SessionCode::SessionCorrupt, message: "fixture execution completed".into(), }) } } #[test] fn full_executor_queue_rejects_before_durable_admission() { let control = Arc::new(Control { work: WorkReceipt { receipt: AdmissionReceipt { session_id: SessionId::new(), submission_id: EntryId::new(), destination: None, }, status: WorkStatus::Queued, cancel_requested: false, }, admissions: AtomicUsize::new(0), aborts: AtomicUsize::new(0), }); let (commands, _command_receiver) = sync_channel(super::COMMAND_QUEUE); for _ in 0..super::COMMAND_QUEUE { let (_reserved, admission) = sync_channel(1); commands .try_send(super::ReservedExecution { request: super::RpcRequest { id: serde_json::Value::Null, method: "prompt".into(), params: serde_json::json!({}), }, admission, }) .unwrap(); } let (_responses, responses) = sync_channel(1); let (waits, _wait_receiver) = sync_channel(1); let mut state = RpcState { control: Arc::clone(&control) as Arc, commands, responses, pending: Arc::new(AtomicUsize::new(0)), admitted: Vec::with_capacity(super::COMMAND_QUEUE), latest_admission: None, waits, }; let response = handle_line( &mut state, r#"{"id":1,"method":"prompt","params":{"input":"not accepted"}}"#, ); assert!(response[0].contains("executor unavailable")); assert_eq!(control.admissions.load(Ordering::SeqCst), 0); } #[test] fn cancellation_receipts_do_not_hide_genuine_executor_error() { for status in [ WorkStatus::Terminal(Some(smith::stream::StopReason::Abort)), WorkStatus::Terminal(Some(smith::stream::StopReason::EndTurn)), WorkStatus::Queued, ] { let submission = EntryId::new(); let control = Arc::new(Control { work: WorkReceipt { receipt: AdmissionReceipt { session_id: SessionId::new(), submission_id: submission, destination: None, }, status, cancel_requested: true, }, admissions: AtomicUsize::new(0), aborts: AtomicUsize::new(0), }); let mut state = RpcState::new( Arc::clone(&control) as Arc, FailingExecutor, ); let start = handle_line( &mut state, r#"{"id":1,"method":"prompt","params":{"input":"aborted"}}"#, ); assert!(start[0].contains("\"event\":\"start\"")); let end = state.recv_line(std::time::Duration::from_secs(1)).unwrap(); assert!(end.contains("TURN_FAILED")); assert!(!end.contains("\"reason\":\"Abort\"")); } } #[test] fn inspection_busy_keyed_retry_and_targeted_abort_remain_available_while_execution_blocks() { let submission = EntryId::new(); let control = Arc::new(Control { work: WorkReceipt { receipt: AdmissionReceipt { session_id: SessionId::new(), submission_id: submission, destination: None, }, status: WorkStatus::Queued, cancel_requested: false, }, admissions: AtomicUsize::new(0), aborts: AtomicUsize::new(0), }); let (started_tx, started_rx) = sync_channel(1); let (release_tx, release_rx) = sync_channel(1); let executions = Arc::new(AtomicUsize::new(0)); let mut state = RpcState::new( Arc::clone(&control) as Arc, BlockingExecutor { started: started_tx, release: release_rx, executions: Arc::clone(&executions), }, ); let start = handle_line( &mut state, r#"{"id":1,"method":"prompt","params":{"input":"blocked","key":"once"}}"#, ); assert!(start[0].contains("\"event\":\"start\"")); assert_eq!( started_rx .recv_timeout(std::time::Duration::from_secs(1)) .unwrap(), submission ); let inspect = handle_line( &mut state, &format!(r#"{{"id":2,"method":"inspect","params":{{"id":"{submission}"}}}}"#), ); assert!(inspect[0].contains(&submission.to_string())); let busy = handle_line( &mut state, r#"{"id":3,"method":"prompt","params":{"input":"other"}}"#, ); assert!(busy[0].contains("\"BUSY\"")); let duplicate = handle_line( &mut state, r#"{"id":4,"method":"prompt","params":{"input":"blocked","key":"once"}}"#, ); assert!(duplicate[0].contains("\"duplicate\":true")); assert_eq!(executions.load(Ordering::SeqCst), 1); let stale = handle_line( &mut state, &format!( r#"{{"id":5,"method":"abort","params":{{"id":"{}"}}}}"#, EntryId::new() ), ); assert!(stale[0].contains("\"STATE\"")); let abort = handle_line( &mut state, &format!(r#"{{"id":6,"method":"abort","params":{{"id":"{submission}"}}}}"#), ); assert!(abort[0].contains(&submission.to_string())); assert_eq!(control.aborts.load(Ordering::SeqCst), 1); release_tx.send(()).unwrap(); let end = state.recv_line(std::time::Duration::from_secs(1)).unwrap(); assert!(end.contains("\"TURN_FAILED\"")); } struct HandoffControl { roots: [EntryId; 3], admitted: AtomicUsize, terminal: AtomicUsize, notifying: std::sync::atomic::AtomicBool, } impl DurableControl for HandoffControl { fn access_mode(&self) -> AccessMode { AccessMode::Writer } fn admit(&self, _: &str, _: Option<&str>) -> Result { let index = self.admitted.fetch_add(1, Ordering::SeqCst); Ok(Admission { work: self.inspect(self.roots[index])?.unwrap(), is_new: true, }) } fn admit_notification( &self, _: EntryId, input: &str, key: Option<&str>, _: smith_core::agent::Delivery, ) -> Result { self.admit(input, key) } fn inspect(&self, id: EntryId) -> Result> { Ok(self .roots .iter() .position(|root| *root == id) .map(|index| WorkReceipt { receipt: AdmissionReceipt { session_id: SessionId::new(), submission_id: id, destination: None, }, status: if self.terminal.load(Ordering::SeqCst) & (1 << index) != 0 { WorkStatus::Terminal(None) } else { WorkStatus::Queued }, cancel_requested: false, })) } fn execution_phase( &self, ) -> Result> { Ok(self.notifying.load(Ordering::SeqCst).then_some(( self.roots[0], smith_core::control::ExecutionPhase::Notifying, ))) } fn abort(&self, _: Option) -> Result { unreachable!() } } struct HandoffExecutor { control: Arc, started: SyncSender, release: Receiver<()>, } impl DurableExecutor for HandoffExecutor { fn execute( &mut self, id: EntryId, _: &mut dyn FnMut(&smith::stream::StreamEvent), ) -> Result { self.started.send(id).unwrap(); self.release .recv_timeout(std::time::Duration::from_secs(5)) .unwrap(); let index = self .control .roots .iter() .position(|root| *root == id) .unwrap(); self.control.terminal.fetch_or(1 << index, Ordering::SeqCst); self.control.notifying.store(false, Ordering::SeqCst); Err(SmithError::Session { code: SessionCode::SessionCorrupt, message: "completed fixture".into(), }) } } #[test] fn implicit_wait_binds_notification_then_oldest_unregistered_admission() { let control = Arc::new(HandoffControl { roots: [EntryId::new(), EntryId::new(), EntryId::new()], admitted: AtomicUsize::new(0), terminal: AtomicUsize::new(0), notifying: std::sync::atomic::AtomicBool::new(false), }); let (started_tx, started_rx) = sync_channel(1); let (release_tx, release_rx) = sync_channel(1); let mut state = RpcState::new( Arc::clone(&control) as Arc, HandoffExecutor { control: Arc::clone(&control), started: started_tx, release: release_rx, }, ); let deadline = std::time::Duration::from_secs(5); assert!( handle_line( &mut state, r#"{"id":1,"method":"prompt","params":{"input":"A"}}"# )[0] .contains("start") ); assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[0]); control.notifying.store(true, Ordering::SeqCst); for id in [2, 3] { assert!( handle_line( &mut state, &format!(r#"{{"id":{id},"method":"prompt","params":{{"input":"queued"}}}}"#) )[0] .contains("start") ); } assert!(handle_line(&mut state, r#"{"id":10,"method":"wait"}"#).is_empty()); release_tx.send(()).unwrap(); assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[1]); assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED")); let notified: serde_json::Value = serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap(); assert_eq!(notified["id"], 10); assert_eq!(notified["settled"], true); // A is unregistered, B has not registered, C is newer: bind B. assert!(handle_line(&mut state, r#"{"id":11,"method":"wait"}"#).is_empty()); let explicit = handle_line( &mut state, &format!( r#"{{"id":12,"method":"wait","params":{{"id":"{}"}}}}"#, control.roots[0] ), ); assert!(explicit[0].contains("true")); release_tx.send(()).unwrap(); assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[2]); assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED")); let oldest: serde_json::Value = serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap(); assert_eq!(oldest["id"], 11); assert_eq!(oldest["settled"], true); release_tx.send(()).unwrap(); assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED")); } #[test] fn previous_completion_retains_future_root_wait_without_reallocating() { let control = Arc::new(HandoffControl { roots: [EntryId::new(), EntryId::new(), EntryId::new()], admitted: AtomicUsize::new(0), terminal: AtomicUsize::new(1), notifying: std::sync::atomic::AtomicBool::new(false), }); let (responses, receiver) = sync_channel(4); let (_wait_sender, waits) = sync_channel(4); let mut worker = super::Worker { control: Arc::clone(&control) as Arc, executor: None, responses, pending: Arc::new(AtomicUsize::new(1)), waits, pending_waits: Vec::with_capacity(super::COMMAND_QUEUE), }; let storage = worker.pending_waits.as_ptr(); worker.pending_waits.push(super::RootWait { request: super::RpcRequest { id: serde_json::json!(1), method: "wait".into(), params: serde_json::json!({}), }, root: control.roots[1], responded: Arc::new(std::sync::atomic::AtomicBool::new(false)), }); worker.respond_settled_waits(control.roots[0]); assert!(receiver.try_recv().is_err()); assert_eq!(worker.pending_waits.len(), 1); assert_eq!(worker.pending_waits.as_ptr(), storage); control.terminal.store(3, Ordering::SeqCst); worker.respond_settled_waits(control.roots[1]); assert_eq!( receiver .recv_timeout(std::time::Duration::from_secs(1)) .unwrap() .record["settled"], true ); assert!(worker.pending_waits.is_empty()); assert_eq!(worker.pending_waits.as_ptr(), storage); } #[test] fn next_admission_wait_survives_previous_completion_drain() { let control = Arc::new(HandoffControl { roots: [EntryId::new(), EntryId::new(), EntryId::new()], admitted: AtomicUsize::new(1), terminal: AtomicUsize::new(0), notifying: std::sync::atomic::AtomicBool::new(false), }); let (commands, command_rx) = sync_channel(super::COMMAND_QUEUE); let (responses, response_rx) = sync_channel(super::RESPONSE_QUEUE); let (waits, wait_rx) = sync_channel(super::COMMAND_QUEUE); let (started_tx, started_rx) = sync_channel(1); let (release_tx, release_rx) = sync_channel(1); let (finished_tx, finished_rx) = sync_channel(1); let (drain_tx, drain_rx) = sync_channel(1); let (drained_tx, drained_rx) = sync_channel(1); let pending = Arc::new(AtomicUsize::new(1)); let mut worker = super::Worker { control: Arc::clone(&control) as Arc, executor: Some(Box::new(HandoffExecutor { control: Arc::clone(&control), started: started_tx, release: release_rx, })), responses, pending: Arc::clone(&pending), waits: wait_rx, pending_waits: Vec::with_capacity(super::COMMAND_QUEUE), }; let root = control.roots[0]; let thread = std::thread::spawn(move || { worker.execute( &super::RpcRequest { id: serde_json::json!(1), method: "prompt".into(), params: serde_json::json!({}), }, root, ); worker.pending.fetch_sub(1, Ordering::SeqCst); finished_tx.send(()).unwrap(); drain_rx .recv_timeout(std::time::Duration::from_secs(5)) .unwrap(); worker.respond_settled_waits(root); drained_tx.send(()).unwrap(); worker.run(command_rx); }); let mut state = RpcState { control: Arc::clone(&control) as Arc, commands, responses: response_rx, pending, admitted: vec![root], latest_admission: Some(root), waits, }; let deadline = std::time::Duration::from_secs(5); assert_eq!(started_rx.recv_timeout(deadline).unwrap(), root); release_tx.send(()).unwrap(); finished_rx.recv_timeout(deadline).unwrap(); assert!(state.is_settled()); assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED")); assert!( handle_line( &mut state, r#"{"id":2,"method":"prompt","params":{"input":"B"}}"# )[0] .contains("start") ); assert!(handle_line(&mut state, r#"{"id":3,"method":"wait"}"#).is_empty()); drain_tx.send(()).unwrap(); drained_rx.recv_timeout(deadline).unwrap(); assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[1]); assert!( state.drain_lines().is_empty(), "A completion must not answer B wait with premature STATE" ); release_tx.send(()).unwrap(); assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED")); let waited: serde_json::Value = serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap(); assert_eq!(waited["id"], 3); assert_eq!(waited["settled"], true); drop(state); thread.join().unwrap(); } }