Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-rpc/src/lib.rs

Raw
//! LF-delimited JSON-RPC over standard input and output (`SMH-SPEC-SPEC0001`,
//! Interface modes).
//!
//! Standard output carries protocol records only.
//! Request IDs are preserved on every response.
//! Durable control remains available while one executor blocks on provider work.

#![deny(unsafe_code)]

use smith::error::Result;
use smith::id::EntryId;
use smith_core::control::{
    AccessMode, DurableControl, DurableExecutor, ExecutionPhase, WorkReceipt,
};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst};
use std::sync::mpsc::{self, Receiver, SyncSender};

/// Commands held for the executor thread.
const COMMAND_QUEUE: usize = 64;
/// Response lines held before the transport drains them.
const RESPONSE_QUEUE: usize = 256;

/// RPC state with independent durable control and blocking execution planes.
pub struct RpcState {
    control: Arc<dyn DurableControl>,
    commands: SyncSender<ReservedExecution>,
    responses: Receiver<RpcResponse>,
    pending: Arc<AtomicUsize>,
    admitted: Vec<EntryId>,
    latest_admission: Option<EntryId>,
    waits: SyncSender<RootWait>,
}

impl RpcState {
    /// Start an RPC session over independently-owned durable contracts.
    #[must_use]
    pub fn new(control: Arc<dyn DurableControl>, executor: impl DurableExecutor + 'static) -> Self {
        Self::with_executor(control, Some(Box::new(executor)))
    }

    /// Start an inspection-only RPC session.
    #[must_use]
    pub fn inspection(control: Arc<dyn DurableControl>) -> Self {
        Self::with_executor(control, None)
    }

    fn with_executor(
        control: Arc<dyn DurableControl>,
        executor: Option<Box<dyn DurableExecutor>>,
    ) -> Self {
        let (command_tx, command_rx) = mpsc::sync_channel(COMMAND_QUEUE);
        let (response_tx, response_rx) = mpsc::sync_channel(RESPONSE_QUEUE);
        let pending = Arc::new(AtomicUsize::new(0));
        let (wait_tx, wait_rx) = mpsc::sync_channel(COMMAND_QUEUE);
        let worker = Worker {
            control: Arc::clone(&control),
            executor,
            responses: response_tx,
            pending: Arc::clone(&pending),
            waits: wait_rx,
            pending_waits: Vec::with_capacity(COMMAND_QUEUE),
        };
        std::thread::spawn(move || worker.run(command_rx));
        Self {
            control,
            commands: command_tx,
            responses: response_rx,
            pending,
            admitted: Vec::with_capacity(COMMAND_QUEUE),
            latest_admission: None,
            waits: wait_tx,
        }
    }

    /// Remove completed asynchronous response lines without blocking.
    #[must_use]
    pub fn drain_lines(&self) -> Vec<String> {
        self.responses
            .try_iter()
            .flat_map(|response| line_or_empty(&response))
            .collect()
    }

    /// Block up to `timeout` for the next response line.
    #[must_use]
    pub fn recv_line(&self, timeout: std::time::Duration) -> Option<String> {
        self.responses.recv_timeout(timeout).ok()?.to_line().ok()
    }

    /// Whether no accepted prompt is executing.
    #[must_use]
    pub fn is_settled(&self) -> bool {
        self.pending.load(SeqCst) == 0
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct RpcRequest {
    id: serde_json::Value,
    method: String,
    params: serde_json::Value,
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct RpcResponse {
    id: serde_json::Value,
    record: serde_json::Value,
}

impl RpcResponse {
    fn to_line(&self) -> Result<String> {
        let mut value = serde_json::json!({"id": self.id});
        merge(&mut value, &self.record);
        let mut line =
            serde_json::to_string(&value).map_err(|error| smith::error::SmithError::Session {
                code: smith::error::SessionCode::RpcEncode,
                message: error.to_string(),
            })?;
        line.push('\n');
        Ok(line)
    }
}

struct ReservedExecution {
    request: RpcRequest,
    admission: Receiver<Option<EntryId>>,
}

struct RootWait {
    request: RpcRequest,
    root: EntryId,
    responded: Arc<AtomicBool>,
}

struct Worker {
    control: Arc<dyn DurableControl>,
    executor: Option<Box<dyn DurableExecutor>>,
    responses: SyncSender<RpcResponse>,
    pending: Arc<AtomicUsize>,
    waits: Receiver<RootWait>,
    pending_waits: Vec<RootWait>,
}

impl Worker {
    #[expect(
        clippy::needless_pass_by_value,
        reason = "worker owns its receiver for the lifetime of its dedicated thread"
    )]
    fn run(mut self, commands: Receiver<ReservedExecution>) {
        while let Ok(ReservedExecution { request, admission }) = commands.recv() {
            if let Ok(Some(id)) = admission.recv() {
                self.execute(&request, id);
                self.pending.fetch_sub(1, SeqCst);
                self.respond_settled_waits(id);
            }
        }
    }

    fn respond_settled_waits(&mut self, root: EntryId) {
        while let Ok(wait) = self.waits.try_recv() {
            if wait.responded.load(SeqCst) {
                continue;
            }
            if self.pending_waits.len() == COMMAND_QUEUE {
                if !wait.responded.swap(true, SeqCst) {
                    self.send(error(&wait.request, "BUSY", "wait queue is full"));
                }
            } else {
                self.pending_waits.push(wait);
            }
        }
        let mut index = 0;
        while index < self.pending_waits.len() {
            if self.pending_waits[index].root == root
                || self.pending_waits[index].responded.load(SeqCst)
            {
                let wait = self.pending_waits.swap_remove(index);
                if !wait.responded.swap(true, SeqCst) {
                    self.send(root_wait_response(
                        self.control.as_ref(),
                        &wait.request,
                        root,
                    ));
                }
            } else {
                index += 1;
            }
        }
    }

    fn execute(&mut self, request: &RpcRequest, id: EntryId) {
        let Some(executor) = self.executor.as_mut() else {
            self.send(RpcResponse {
                id: request.id.clone(),
                record: serde_json::json!({"event": "end", "error": {"code": "STATE", "message": "inspection mode has no executor"}}),
            });
            return;
        };
        let responses = &self.responses;
        let mut observe = |event: &smith::stream::StreamEvent| {
            if let smith::stream::StreamEvent::TextDelta { delta, .. } = event {
                let _send_result = responses.send(result(
                    request,
                    serde_json::json!({"event": "delta", "text": delta}),
                ));
            }
        };
        match executor.execute(id, &mut observe) {
            Ok(outcome) => {
                self.send(result(
                    request,
                    serde_json::json!({
                        "event": "end",
                        "text": outcome.text,
                        "reason": format!("{:?}", outcome.reason),
                        "cost": outcome.cost.map(|cost| serde_json::json!({
                            "input_tokens": cost.input_tokens,
                            "output_tokens": cost.output_tokens,
                        })),
                        "cost_complete": outcome.cost_complete,
                    }),
                ));
            }
            Err(error) => self.send(RpcResponse {
                id: request.id.clone(),
                record: serde_json::json!({
                    "event": "end",
                    "error": {"code": "TURN_FAILED", "message": error.to_string()},
                }),
            }),
        }
    }

    fn send(&self, response: RpcResponse) {
        let _send_result = self.responses.send(response);
    }
}

/// Handle one input line, returning immediate records.
#[must_use]
pub fn handle_line(state: &mut RpcState, line: &str) -> Vec<String> {
    let trimmed = line.trim();
    if trimmed.is_empty() {
        return Vec::new();
    }
    let parsed: serde_json::Value = match serde_json::from_str(trimmed) {
        Ok(value) => value,
        Err(error) => {
            return line_or_empty(&RpcResponse {
                id: serde_json::Value::Null,
                record: serde_json::json!({"error": {"code": "PARSE", "message": error.to_string()}}),
            });
        }
    };
    let request = RpcRequest {
        id: parsed.get("id").cloned().unwrap_or(serde_json::Value::Null),
        method: parsed
            .get("method")
            .and_then(serde_json::Value::as_str)
            .unwrap_or_default()
            .to_string(),
        params: parsed
            .get("params")
            .cloned()
            .unwrap_or_else(|| serde_json::json!({})),
    };
    dispatch(state, &request)
        .into_iter()
        .flat_map(|response| line_or_empty(&response))
        .collect()
}

fn dispatch(state: &mut RpcState, request: &RpcRequest) -> Vec<RpcResponse> {
    match request.method.as_str() {
        "ping" => vec![result(request, serde_json::json!({"pong": true}))],
        "prompt" => start_prompt(state, request),
        "abort" => abort(state, request),
        "inspect" => inspect(state, request),
        "status" => vec![status(state, request)],
        "wait" => wait(state, request),
        other => vec![error(
            request,
            "METHOD_UNKNOWN",
            &format!("unknown method {other}"),
        )],
    }
}

fn start_prompt(state: &mut RpcState, request: &RpcRequest) -> Vec<RpcResponse> {
    let input = request
        .params
        .get("input")
        .and_then(serde_json::Value::as_str)
        .unwrap_or_default();
    let key = request
        .params
        .get("key")
        .and_then(serde_json::Value::as_str);
    let delivery = match request
        .params
        .get("delivery")
        .and_then(serde_json::Value::as_str)
    {
        None | Some("AfterTurn") => smith_core::agent::Delivery::AfterTurn,
        Some("BetweenRequests") => smith_core::agent::Delivery::BetweenRequests,
        Some(_) => {
            return vec![error(
                request,
                "PARAMS",
                "delivery must be BetweenRequests or AfterTurn",
            )];
        }
    };
    let notification = match state.control.execution_phase() {
        Ok(Some((root, ExecutionPhase::Notifying))) => Some(root),
        Ok(_) => None,
        Err(fault) => return vec![error(request, "STATE", &fault.to_string())],
    };
    if !state.is_settled() && notification.is_none() && key.is_none() {
        return vec![error(request, "BUSY", "a prompt is already running")];
    }
    // Reserve bounded executor ownership BEFORE durable acceptance. A rollback
    // sends None, never an executable identity or a fabricated accepted receipt.
    state.admitted.retain(|id| {
        !matches!(state.control.inspect(*id), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_)))
    });
    if state.pending.load(SeqCst) >= COMMAND_QUEUE || state.admitted.len() >= COMMAND_QUEUE {
        return vec![error(request, "BUSY", "executor queue is full")];
    }
    let (accepted, admission) = mpsc::sync_channel(1);
    if state
        .commands
        .try_send(ReservedExecution {
            request: request.clone(),
            admission,
        })
        .is_err()
    {
        return vec![error(request, "STATE", "executor unavailable")];
    }
    let admission = match notification {
        Some(root) => state.control.admit_notification(root, input, key, delivery),
        None => state.control.admit(input, key),
    };
    let admission = match admission {
        Ok(admission) => admission,
        Err(control_error) => {
            let _rollback = accepted.send(None);
            return vec![error(
                request,
                if state.is_settled() { "STATE" } else { "BUSY" },
                &control_error.to_string(),
            )];
        }
    };
    let receipt = receipt(&admission.work);
    if !admission.is_new {
        let _rollback = accepted.send(None);
        return vec![result(
            request,
            serde_json::json!({"event": "start", "duplicate": true, "receipt": receipt}),
        )];
    }
    state.admitted.push(admission.work.receipt.submission_id);
    state.latest_admission = Some(admission.work.receipt.submission_id);
    state.pending.fetch_add(1, SeqCst);
    if accepted
        .send(Some(admission.work.receipt.submission_id))
        .is_err()
    {
        state.pending.fetch_sub(1, SeqCst);
        return vec![error(
            request,
            "STATE",
            "executor unavailable; admitted work remains queued",
        )];
    }
    vec![result(
        request,
        serde_json::json!({"event": "start", "receipt": receipt}),
    )]
}

fn abort(state: &RpcState, request: &RpcRequest) -> Vec<RpcResponse> {
    let id = match request.params.get("id") {
        Some(value) => match stable_id(value) {
            Ok(id) => Some(id),
            Err(response) => return vec![error(request, "PARAMS", response)],
        },
        None => None,
    };
    match state.control.abort(id) {
        Ok(id) => vec![result(
            request,
            serde_json::json!({"aborting": true, "submission_id": id.to_string()}),
        )],
        Err(control_error) => vec![error(request, "STATE", &control_error.to_string())],
    }
}

fn inspect(state: &RpcState, request: &RpcRequest) -> Vec<RpcResponse> {
    let Some(value) = request.params.get("id") else {
        return vec![error(request, "PARAMS", "id is required")];
    };
    let id = match stable_id(value) {
        Ok(id) => id,
        Err(message) => return vec![error(request, "PARAMS", message)],
    };
    match state.control.inspect(id) {
        Ok(Some(work)) => vec![result(
            request,
            serde_json::json!({"receipt": receipt(&work)}),
        )],
        Ok(None) => vec![result(request, serde_json::json!({"receipt": null}))],
        Err(control_error) => vec![error(request, "STATE", &control_error.to_string())],
    }
}

fn stable_id(value: &serde_json::Value) -> std::result::Result<EntryId, &'static str> {
    let Some(value) = value.as_str() else {
        return Err("id is not a stable entry identity");
    };
    serde_json::from_value(serde_json::Value::String(value.to_owned()))
        .map_err(|_| "id is not a stable entry identity")
}

fn status(state: &RpcState, request: &RpcRequest) -> RpcResponse {
    let phase = match state.control.execution_phase() {
        Ok(phase) => phase,
        Err(fault) => return error(request, "STATE", &fault.to_string()),
    };
    let mut settled = state.is_settled();
    let mut active = phase.map(|(id, _)| id);
    if settled && let Some(id) = state.latest_admission {
        match state.control.inspect(id) {
            Ok(Some(work)) => {
                settled = matches!(work.status, smith_core::store::WorkStatus::Terminal(_));
                if work.status == smith_core::store::WorkStatus::Running {
                    active = Some(id);
                }
            }
            Ok(None) => settled = false,
            Err(fault) => return error(request, "STATE", &fault.to_string()),
        }
    }
    result(
        request,
        serde_json::json!({
            "settled": settled,
            "access_mode": access_mode(state.control.access_mode()),
            "active_submission_id": active.map(|id| id.to_string()),
        }),
    )
}

fn wait(state: &RpcState, request: &RpcRequest) -> Vec<RpcResponse> {
    let explicit = match request.params.get("id") {
        Some(value) => match stable_id(value) {
            Ok(id) => Some(id),
            Err(message) => return vec![error(request, "PARAMS", message)],
        },
        None => None,
    };
    let phase = match state.control.execution_phase() {
        Ok(phase) => phase,
        Err(fault) => return vec![error(request, "STATE", &fault.to_string())],
    };
    // Notification ownership wins until cleanup ends; otherwise bind the oldest
    // nonterminal admission, including the executor registration handoff.
    let root = explicit
        .or_else(|| phase.map(|(id, _)| id))
        .or_else(|| state.admitted.iter().copied().find(|id| {
            !matches!(state.control.inspect(*id), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_)))
        }))
        .or(state.latest_admission);
    let Some(root) = root else {
        return vec![result(request, serde_json::json!({"settled": true}))];
    };
    if state.is_settled()
        || matches!(state.control.inspect(root), Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_)))
    {
        return vec![root_wait_response(state.control.as_ref(), request, root)];
    }
    let responded = Arc::new(AtomicBool::new(false));
    if state
        .waits
        .try_send(RootWait {
            request: request.clone(),
            root,
            responded: Arc::clone(&responded),
        })
        .is_err()
    {
        return vec![error(request, "STATE", "wait queue unavailable")];
    }
    // Close the enqueue-versus-cleanup race: either worker sees this waiter,
    // or its receipt is terminal now. The response lease prevents duplication.
    if let Ok(Some(work)) = state.control.inspect(root)
        && matches!(work.status, smith_core::store::WorkStatus::Terminal(_))
        && !responded.swap(true, SeqCst)
    {
        return vec![result(request, serde_json::json!({"settled": true}))];
    }
    Vec::new()
}

fn root_wait_response(
    control: &dyn DurableControl,
    request: &RpcRequest,
    root: EntryId,
) -> RpcResponse {
    match control.inspect(root) {
        Ok(Some(work)) if matches!(work.status, smith_core::store::WorkStatus::Terminal(_)) => {
            result(request, serde_json::json!({"settled": true}))
        }
        Ok(_) => error(
            request,
            "STATE",
            "root cleanup finished without durable terminal acknowledgement",
        ),
        Err(fault) => error(request, "STATE", &fault.to_string()),
    }
}

fn receipt(work: &WorkReceipt) -> serde_json::Value {
    serde_json::json!({
        "session_id": work.receipt.session_id.to_string(),
        "submission_id": work.receipt.submission_id.to_string(),
        "destination": work.receipt.destination.map(|id| id.to_string()),
        "status": format!("{:?}", work.status),
        "cancel_requested": work.cancel_requested,
    })
}

const fn access_mode(mode: AccessMode) -> &'static str {
    match mode {
        AccessMode::ReadOnly => "read_only",
        AccessMode::Writer => "writer",
    }
}

fn merge(target: &mut serde_json::Value, patch: &serde_json::Value) {
    if let (Some(target), Some(patch)) = (target.as_object_mut(), patch.as_object()) {
        target.extend(
            patch
                .iter()
                .map(|(key, value)| (key.clone(), value.clone())),
        );
    }
}

fn line_or_empty(response: &RpcResponse) -> Vec<String> {
    response
        .to_line()
        .map_or_else(|_| Vec::new(), |line| vec![line])
}

fn result(request: &RpcRequest, record: serde_json::Value) -> RpcResponse {
    RpcResponse {
        id: request.id.clone(),
        record,
    }
}

fn error(request: &RpcRequest, code: &str, message: &str) -> RpcResponse {
    RpcResponse {
        id: request.id.clone(),
        record: serde_json::json!({"error": {"code": code, "message": message}}),
    }
}

#[cfg(test)]
mod tests {
    use super::{RpcState, handle_line};
    use smith::error::{Result, SessionCode, SmithError};
    use smith::id::{EntryId, SessionId};
    use smith_core::control::{
        AccessMode, Admission, DurableControl, DurableExecutor, WorkReceipt,
    };
    use smith_core::store::{AdmissionReceipt, WorkStatus};
    use std::sync::Arc;
    use std::sync::atomic::{AtomicUsize, Ordering};
    use std::sync::mpsc::{Receiver, SyncSender, sync_channel};

    struct Control {
        work: WorkReceipt,
        admissions: AtomicUsize,
        aborts: AtomicUsize,
    }

    impl DurableControl for Control {
        fn access_mode(&self) -> AccessMode {
            AccessMode::Writer
        }

        fn admit(&self, _: &str, key: Option<&str>) -> Result<Admission> {
            let count = self.admissions.fetch_add(1, Ordering::SeqCst);
            Ok(Admission {
                work: self.work,
                is_new: count == 0 || key.is_none(),
            })
        }

        fn inspect(&self, id: EntryId) -> Result<Option<WorkReceipt>> {
            Ok((id == self.work.receipt.submission_id).then_some(self.work))
        }

        fn abort(&self, id: Option<EntryId>) -> Result<EntryId> {
            let id = id.unwrap_or(self.work.receipt.submission_id);
            if id != self.work.receipt.submission_id {
                return Err(SmithError::Session {
                    code: SessionCode::SessionCorrupt,
                    message: "stale submission".into(),
                });
            }
            self.aborts.fetch_add(1, Ordering::SeqCst);
            Ok(id)
        }
    }

    struct FailingExecutor;

    impl DurableExecutor for FailingExecutor {
        fn execute(
            &mut self,
            _id: EntryId,
            _observe: &mut dyn FnMut(&smith::stream::StreamEvent),
        ) -> Result<smith_core::agent::TurnOutcome> {
            Err(SmithError::Session {
                code: SessionCode::SessionCorrupt,
                message: "fixture execution failed".into(),
            })
        }
    }

    struct BlockingExecutor {
        started: SyncSender<EntryId>,
        release: Receiver<()>,
        executions: Arc<AtomicUsize>,
    }

    impl DurableExecutor for BlockingExecutor {
        fn execute(
            &mut self,
            id: EntryId,
            _observe: &mut dyn FnMut(&smith::stream::StreamEvent),
        ) -> Result<smith_core::agent::TurnOutcome> {
            self.executions.fetch_add(1, Ordering::SeqCst);
            self.started.send(id).map_err(|error| SmithError::Session {
                code: SessionCode::SessionCorrupt,
                message: error.to_string(),
            })?;
            self.release.recv().map_err(|error| SmithError::Session {
                code: SessionCode::SessionCorrupt,
                message: error.to_string(),
            })?;
            Err(SmithError::Session {
                code: SessionCode::SessionCorrupt,
                message: "fixture execution completed".into(),
            })
        }
    }

    #[test]
    fn full_executor_queue_rejects_before_durable_admission() {
        let control = Arc::new(Control {
            work: WorkReceipt {
                receipt: AdmissionReceipt {
                    session_id: SessionId::new(),
                    submission_id: EntryId::new(),
                    destination: None,
                },
                status: WorkStatus::Queued,
                cancel_requested: false,
            },
            admissions: AtomicUsize::new(0),
            aborts: AtomicUsize::new(0),
        });
        let (commands, _command_receiver) = sync_channel(super::COMMAND_QUEUE);
        for _ in 0..super::COMMAND_QUEUE {
            let (_reserved, admission) = sync_channel(1);
            commands
                .try_send(super::ReservedExecution {
                    request: super::RpcRequest {
                        id: serde_json::Value::Null,
                        method: "prompt".into(),
                        params: serde_json::json!({}),
                    },
                    admission,
                })
                .unwrap();
        }
        let (_responses, responses) = sync_channel(1);
        let (waits, _wait_receiver) = sync_channel(1);
        let mut state = RpcState {
            control: Arc::clone(&control) as Arc<dyn DurableControl>,
            commands,
            responses,
            pending: Arc::new(AtomicUsize::new(0)),
            admitted: Vec::with_capacity(super::COMMAND_QUEUE),
            latest_admission: None,
            waits,
        };
        let response = handle_line(
            &mut state,
            r#"{"id":1,"method":"prompt","params":{"input":"not accepted"}}"#,
        );
        assert!(response[0].contains("executor unavailable"));
        assert_eq!(control.admissions.load(Ordering::SeqCst), 0);
    }

    #[test]
    fn cancellation_receipts_do_not_hide_genuine_executor_error() {
        for status in [
            WorkStatus::Terminal(Some(smith::stream::StopReason::Abort)),
            WorkStatus::Terminal(Some(smith::stream::StopReason::EndTurn)),
            WorkStatus::Queued,
        ] {
            let submission = EntryId::new();
            let control = Arc::new(Control {
                work: WorkReceipt {
                    receipt: AdmissionReceipt {
                        session_id: SessionId::new(),
                        submission_id: submission,
                        destination: None,
                    },
                    status,
                    cancel_requested: true,
                },
                admissions: AtomicUsize::new(0),
                aborts: AtomicUsize::new(0),
            });
            let mut state = RpcState::new(
                Arc::clone(&control) as Arc<dyn DurableControl>,
                FailingExecutor,
            );
            let start = handle_line(
                &mut state,
                r#"{"id":1,"method":"prompt","params":{"input":"aborted"}}"#,
            );
            assert!(start[0].contains("\"event\":\"start\""));
            let end = state.recv_line(std::time::Duration::from_secs(1)).unwrap();
            assert!(end.contains("TURN_FAILED"));
            assert!(!end.contains("\"reason\":\"Abort\""));
        }
    }

    #[test]
    fn inspection_busy_keyed_retry_and_targeted_abort_remain_available_while_execution_blocks() {
        let submission = EntryId::new();
        let control = Arc::new(Control {
            work: WorkReceipt {
                receipt: AdmissionReceipt {
                    session_id: SessionId::new(),
                    submission_id: submission,
                    destination: None,
                },
                status: WorkStatus::Queued,
                cancel_requested: false,
            },
            admissions: AtomicUsize::new(0),
            aborts: AtomicUsize::new(0),
        });
        let (started_tx, started_rx) = sync_channel(1);
        let (release_tx, release_rx) = sync_channel(1);
        let executions = Arc::new(AtomicUsize::new(0));
        let mut state = RpcState::new(
            Arc::clone(&control) as Arc<dyn DurableControl>,
            BlockingExecutor {
                started: started_tx,
                release: release_rx,
                executions: Arc::clone(&executions),
            },
        );

        let start = handle_line(
            &mut state,
            r#"{"id":1,"method":"prompt","params":{"input":"blocked","key":"once"}}"#,
        );
        assert!(start[0].contains("\"event\":\"start\""));
        assert_eq!(
            started_rx
                .recv_timeout(std::time::Duration::from_secs(1))
                .unwrap(),
            submission
        );
        let inspect = handle_line(
            &mut state,
            &format!(r#"{{"id":2,"method":"inspect","params":{{"id":"{submission}"}}}}"#),
        );
        assert!(inspect[0].contains(&submission.to_string()));
        let busy = handle_line(
            &mut state,
            r#"{"id":3,"method":"prompt","params":{"input":"other"}}"#,
        );
        assert!(busy[0].contains("\"BUSY\""));
        let duplicate = handle_line(
            &mut state,
            r#"{"id":4,"method":"prompt","params":{"input":"blocked","key":"once"}}"#,
        );
        assert!(duplicate[0].contains("\"duplicate\":true"));
        assert_eq!(executions.load(Ordering::SeqCst), 1);
        let stale = handle_line(
            &mut state,
            &format!(
                r#"{{"id":5,"method":"abort","params":{{"id":"{}"}}}}"#,
                EntryId::new()
            ),
        );
        assert!(stale[0].contains("\"STATE\""));
        let abort = handle_line(
            &mut state,
            &format!(r#"{{"id":6,"method":"abort","params":{{"id":"{submission}"}}}}"#),
        );
        assert!(abort[0].contains(&submission.to_string()));
        assert_eq!(control.aborts.load(Ordering::SeqCst), 1);
        release_tx.send(()).unwrap();
        let end = state.recv_line(std::time::Duration::from_secs(1)).unwrap();
        assert!(end.contains("\"TURN_FAILED\""));
    }
    struct HandoffControl {
        roots: [EntryId; 3],
        admitted: AtomicUsize,
        terminal: AtomicUsize,
        notifying: std::sync::atomic::AtomicBool,
    }
    impl DurableControl for HandoffControl {
        fn access_mode(&self) -> AccessMode {
            AccessMode::Writer
        }
        fn admit(&self, _: &str, _: Option<&str>) -> Result<Admission> {
            let index = self.admitted.fetch_add(1, Ordering::SeqCst);
            Ok(Admission {
                work: self.inspect(self.roots[index])?.unwrap(),
                is_new: true,
            })
        }
        fn admit_notification(
            &self,
            _: EntryId,
            input: &str,
            key: Option<&str>,
            _: smith_core::agent::Delivery,
        ) -> Result<Admission> {
            self.admit(input, key)
        }
        fn inspect(&self, id: EntryId) -> Result<Option<WorkReceipt>> {
            Ok(self
                .roots
                .iter()
                .position(|root| *root == id)
                .map(|index| WorkReceipt {
                    receipt: AdmissionReceipt {
                        session_id: SessionId::new(),
                        submission_id: id,
                        destination: None,
                    },
                    status: if self.terminal.load(Ordering::SeqCst) & (1 << index) != 0 {
                        WorkStatus::Terminal(None)
                    } else {
                        WorkStatus::Queued
                    },
                    cancel_requested: false,
                }))
        }
        fn execution_phase(
            &self,
        ) -> Result<Option<(EntryId, smith_core::control::ExecutionPhase)>> {
            Ok(self.notifying.load(Ordering::SeqCst).then_some((
                self.roots[0],
                smith_core::control::ExecutionPhase::Notifying,
            )))
        }
        fn abort(&self, _: Option<EntryId>) -> Result<EntryId> {
            unreachable!()
        }
    }
    struct HandoffExecutor {
        control: Arc<HandoffControl>,
        started: SyncSender<EntryId>,
        release: Receiver<()>,
    }
    impl DurableExecutor for HandoffExecutor {
        fn execute(
            &mut self,
            id: EntryId,
            _: &mut dyn FnMut(&smith::stream::StreamEvent),
        ) -> Result<smith_core::agent::TurnOutcome> {
            self.started.send(id).unwrap();
            self.release
                .recv_timeout(std::time::Duration::from_secs(5))
                .unwrap();
            let index = self
                .control
                .roots
                .iter()
                .position(|root| *root == id)
                .unwrap();
            self.control.terminal.fetch_or(1 << index, Ordering::SeqCst);
            self.control.notifying.store(false, Ordering::SeqCst);
            Err(SmithError::Session {
                code: SessionCode::SessionCorrupt,
                message: "completed fixture".into(),
            })
        }
    }
    #[test]
    fn implicit_wait_binds_notification_then_oldest_unregistered_admission() {
        let control = Arc::new(HandoffControl {
            roots: [EntryId::new(), EntryId::new(), EntryId::new()],
            admitted: AtomicUsize::new(0),
            terminal: AtomicUsize::new(0),
            notifying: std::sync::atomic::AtomicBool::new(false),
        });
        let (started_tx, started_rx) = sync_channel(1);
        let (release_tx, release_rx) = sync_channel(1);
        let mut state = RpcState::new(
            Arc::clone(&control) as Arc<dyn DurableControl>,
            HandoffExecutor {
                control: Arc::clone(&control),
                started: started_tx,
                release: release_rx,
            },
        );
        let deadline = std::time::Duration::from_secs(5);
        assert!(
            handle_line(
                &mut state,
                r#"{"id":1,"method":"prompt","params":{"input":"A"}}"#
            )[0]
            .contains("start")
        );
        assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[0]);
        control.notifying.store(true, Ordering::SeqCst);
        for id in [2, 3] {
            assert!(
                handle_line(
                    &mut state,
                    &format!(r#"{{"id":{id},"method":"prompt","params":{{"input":"queued"}}}}"#)
                )[0]
                .contains("start")
            );
        }
        assert!(handle_line(&mut state, r#"{"id":10,"method":"wait"}"#).is_empty());
        release_tx.send(()).unwrap();
        assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[1]);
        assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED"));
        let notified: serde_json::Value =
            serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap();
        assert_eq!(notified["id"], 10);
        assert_eq!(notified["settled"], true);
        // A is unregistered, B has not registered, C is newer: bind B.
        assert!(handle_line(&mut state, r#"{"id":11,"method":"wait"}"#).is_empty());
        let explicit = handle_line(
            &mut state,
            &format!(
                r#"{{"id":12,"method":"wait","params":{{"id":"{}"}}}}"#,
                control.roots[0]
            ),
        );
        assert!(explicit[0].contains("true"));
        release_tx.send(()).unwrap();
        assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[2]);
        assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED"));
        let oldest: serde_json::Value =
            serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap();
        assert_eq!(oldest["id"], 11);
        assert_eq!(oldest["settled"], true);
        release_tx.send(()).unwrap();
        assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED"));
    }

    #[test]
    fn previous_completion_retains_future_root_wait_without_reallocating() {
        let control = Arc::new(HandoffControl {
            roots: [EntryId::new(), EntryId::new(), EntryId::new()],
            admitted: AtomicUsize::new(0),
            terminal: AtomicUsize::new(1),
            notifying: std::sync::atomic::AtomicBool::new(false),
        });
        let (responses, receiver) = sync_channel(4);
        let (_wait_sender, waits) = sync_channel(4);
        let mut worker = super::Worker {
            control: Arc::clone(&control) as Arc<dyn DurableControl>,
            executor: None,
            responses,
            pending: Arc::new(AtomicUsize::new(1)),
            waits,
            pending_waits: Vec::with_capacity(super::COMMAND_QUEUE),
        };
        let storage = worker.pending_waits.as_ptr();
        worker.pending_waits.push(super::RootWait {
            request: super::RpcRequest {
                id: serde_json::json!(1),
                method: "wait".into(),
                params: serde_json::json!({}),
            },
            root: control.roots[1],
            responded: Arc::new(std::sync::atomic::AtomicBool::new(false)),
        });
        worker.respond_settled_waits(control.roots[0]);
        assert!(receiver.try_recv().is_err());
        assert_eq!(worker.pending_waits.len(), 1);
        assert_eq!(worker.pending_waits.as_ptr(), storage);
        control.terminal.store(3, Ordering::SeqCst);
        worker.respond_settled_waits(control.roots[1]);
        assert_eq!(
            receiver
                .recv_timeout(std::time::Duration::from_secs(1))
                .unwrap()
                .record["settled"],
            true
        );
        assert!(worker.pending_waits.is_empty());
        assert_eq!(worker.pending_waits.as_ptr(), storage);
    }
    #[test]
    fn next_admission_wait_survives_previous_completion_drain() {
        let control = Arc::new(HandoffControl {
            roots: [EntryId::new(), EntryId::new(), EntryId::new()],
            admitted: AtomicUsize::new(1),
            terminal: AtomicUsize::new(0),
            notifying: std::sync::atomic::AtomicBool::new(false),
        });
        let (commands, command_rx) = sync_channel(super::COMMAND_QUEUE);
        let (responses, response_rx) = sync_channel(super::RESPONSE_QUEUE);
        let (waits, wait_rx) = sync_channel(super::COMMAND_QUEUE);
        let (started_tx, started_rx) = sync_channel(1);
        let (release_tx, release_rx) = sync_channel(1);
        let (finished_tx, finished_rx) = sync_channel(1);
        let (drain_tx, drain_rx) = sync_channel(1);
        let (drained_tx, drained_rx) = sync_channel(1);
        let pending = Arc::new(AtomicUsize::new(1));
        let mut worker = super::Worker {
            control: Arc::clone(&control) as Arc<dyn DurableControl>,
            executor: Some(Box::new(HandoffExecutor {
                control: Arc::clone(&control),
                started: started_tx,
                release: release_rx,
            })),
            responses,
            pending: Arc::clone(&pending),
            waits: wait_rx,
            pending_waits: Vec::with_capacity(super::COMMAND_QUEUE),
        };
        let root = control.roots[0];
        let thread = std::thread::spawn(move || {
            worker.execute(
                &super::RpcRequest {
                    id: serde_json::json!(1),
                    method: "prompt".into(),
                    params: serde_json::json!({}),
                },
                root,
            );
            worker.pending.fetch_sub(1, Ordering::SeqCst);
            finished_tx.send(()).unwrap();
            drain_rx
                .recv_timeout(std::time::Duration::from_secs(5))
                .unwrap();
            worker.respond_settled_waits(root);
            drained_tx.send(()).unwrap();
            worker.run(command_rx);
        });
        let mut state = RpcState {
            control: Arc::clone(&control) as Arc<dyn DurableControl>,
            commands,
            responses: response_rx,
            pending,
            admitted: vec![root],
            latest_admission: Some(root),
            waits,
        };
        let deadline = std::time::Duration::from_secs(5);
        assert_eq!(started_rx.recv_timeout(deadline).unwrap(), root);
        release_tx.send(()).unwrap();
        finished_rx.recv_timeout(deadline).unwrap();
        assert!(state.is_settled());
        assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED"));
        assert!(
            handle_line(
                &mut state,
                r#"{"id":2,"method":"prompt","params":{"input":"B"}}"#
            )[0]
            .contains("start")
        );
        assert!(handle_line(&mut state, r#"{"id":3,"method":"wait"}"#).is_empty());
        drain_tx.send(()).unwrap();
        drained_rx.recv_timeout(deadline).unwrap();
        assert_eq!(started_rx.recv_timeout(deadline).unwrap(), control.roots[1]);
        assert!(
            state.drain_lines().is_empty(),
            "A completion must not answer B wait with premature STATE"
        );
        release_tx.send(()).unwrap();
        assert!(state.recv_line(deadline).unwrap().contains("TURN_FAILED"));
        let waited: serde_json::Value =
            serde_json::from_str(&state.recv_line(deadline).unwrap()).unwrap();
        assert_eq!(waited["id"], 3);
        assert_eq!(waited["settled"], true);
        drop(state);
        thread.join().unwrap();
    }
}