Luigit
repositories / smith

smith

There are many coding harnesses - but this one is fast

owned by admin

smith-core/src/store/payload_tests.rs

Raw
//! SMH-SPEC-SPEC0001: same-ledger complete payloads, bounded reads, ownership, recovery.
use super::*;
use crate::frame::{NestedPayloadChunk, NestedPayloadPhase};
use crate::metadata::{
    self, NestedEffectCompletion, NestedEffectIntent, NestedPayload, NestedPayloadRef,
};
use crate::session::EntryContent;
use smith::error::MutationState;

fn frames(input: &str, output: &str) -> (PathBuf, OpenSession, Vec<Frame>, Frame) {
    let (path, opened, mut outer) = super::commit_tests::setup(MutationState::Applied);
    let Frame::Commit {
        checkpoint,
        entries,
    } = &mut outer
    else {
        unreachable!()
    };
    let payload = |text: &str| {
        NestedPayload::Chunks(NestedPayloadRef {
            payload_bytes: text.len().try_into().unwrap(),
        })
    };
    let intent = EntryFrame::new(
        entries[0].parent,
        &EntryContent::Meta {
            kind: metadata::NESTED_INTENT_KIND.into(),
            detail: metadata::encode(&NestedEffectIntent {
                submission_id: checkpoint.submission_id,
                call_id: checkpoint.call_id,
                index: 0,
                operation: "http",
                component: None,
                input: payload(input),
            })
            .unwrap(),
        },
    )
    .unwrap();
    let completion = EntryFrame::new(
        Some(intent.id),
        &EntryContent::Meta {
            kind: metadata::NESTED_COMPLETION_KIND.into(),
            detail: metadata::encode(&NestedEffectCompletion {
                submission_id: checkpoint.submission_id,
                call_id: checkpoint.call_id,
                index: 0,
                ok: true,
                mutation_state: MutationState::Applied,
                output: payload(output),
            })
            .unwrap(),
        },
    )
    .unwrap();
    let mut frames = Vec::new();
    for (text, phase) in [
        (input, NestedPayloadPhase::Input),
        (output, NestedPayloadPhase::Output),
    ] {
        for (ordinal, bytes) in text
            .as_bytes()
            .chunks(metadata::NESTED_CHUNK_BYTES_MAX)
            .enumerate()
        {
            frames.push(Frame::NestedPayload {
                chunk: Box::new(NestedPayloadChunk {
                    submission_id: checkpoint.submission_id,
                    call_id: checkpoint.call_id,
                    effect_id: intent.id,
                    index: 0,
                    phase,
                    ordinal: ordinal.try_into().unwrap(),
                    bytes: bytes.to_vec(),
                }),
            });
        }
        frames.push(if phase == NestedPayloadPhase::Input {
            Frame::Known {
                entry: intent.clone(),
            }
        } else {
            Frame::NestedCommit {
                entry: completion.clone(),
            }
        });
    }
    entries[0].parent = Some(completion.id);
    (path, opened, frames, outer)
}

#[test]
fn full_payloads_reconstruct_exactly_without_resident_history_payloads() {
    // Deliberately split UTF-8 scalars across physical boundaries.
    let input = "a🦀".repeat(14000);
    let output = "bλ".repeat(350000);
    let (path, mut opened, frames, outer) = frames(&input, &output);
    let input_id = frames.iter().flat_map(Frame::entries).next().unwrap().id;
    let output_id = frames.iter().flat_map(Frame::entries).last().unwrap().id;
    for frame in &frames {
        if let Frame::NestedCommit { entry } = frame {
            let intent_id = entry.parent.unwrap();
            let (_, intent_offset) = SessionReader::open(&path)
                .unwrap()
                .find_entry(intent_id, Config::default_valid().frame_bytes_max)
                .unwrap()
                .unwrap();
            opened
                .writer
                .append_frame(&Frame::Checkpoint {
                    id: intent_id,
                    offset: intent_offset,
                })
                .unwrap();
        }
        opened.writer.append_frame(frame).unwrap();
    }
    opened.writer.append_frame(&outer).unwrap();
    opened.writer.select_entry(output_id).unwrap();
    drop(opened);
    let limit = Config::default_valid().frame_bytes_max;
    let reader = SessionReader::open(&path).unwrap();
    assert_eq!(reader.nested_payload(input_id, limit).unwrap(), input);
    assert_eq!(reader.nested_payload(output_id, limit).unwrap(), output);
    let session = reader.selected_session(limit).unwrap();
    assert!(
        session
            .active_branch()
            .entries()
            .iter()
            .map(|entry| session.bytes(entry).len())
            .sum::<usize>()
            < 16 * 1024,
        "resident history contains references, not large canonical payloads"
    );
    reader.verify_full(limit).unwrap();
    drop(reader);
    repair_session(&path, limit).unwrap();
    let reader = SessionReader::open(&path).unwrap();
    reader.verify_full(limit).unwrap();
    assert_eq!(reader.selected_tip(limit).unwrap(), Some(output_id));
    assert_eq!(reader.nested_payload(output_id, limit).unwrap(), output);
    drop(reader);
    super::commit_tests::clear(&path);
}

#[test]
fn origin_boundaries_are_bounded_matched_and_never_appended_by_live_writers() {
    assert_eq!(std::mem::size_of::<OriginScopes>(), 40);
    let config = Config::default_valid();
    for mode in ["unclosed", "unopened", "mismatch", "depth", "transition"] {
        let (path, mut opened, _) = super::commit_tests::setup(MutationState::Applied);
        let source = SessionId::new();
        let begin = Frame::Origin {
            source,
            begin: true,
        };
        assert!(opened.writer.append_frame(&begin).is_err());
        let frames = match mode {
            "unclosed" => vec![begin],
            "unopened" => vec![Frame::Origin {
                source,
                begin: false,
            }],
            "mismatch" => vec![
                begin,
                Frame::Origin {
                    source: SessionId::new(),
                    begin: false,
                },
            ],
            "depth" => vec![begin; ORIGIN_DEPTH_MAX + 1],
            _ => vec![
                begin,
                Frame::Claim {
                    submission_id: EntryId::new(),
                },
                Frame::Origin {
                    source,
                    begin: false,
                },
            ],
        };
        let mut file = OpenOptions::new().append(true).open(&path).unwrap();
        for frame in &frames {
            write_frame(&mut file, frame, config.frame_bytes_max).unwrap();
        }
        drop(file);
        assert!(
            SessionReader::open(&path)
                .unwrap()
                .validate_work(config.frame_bytes_max)
                .is_err(),
            "{mode}"
        );
        let mut complete = Vec::new();
        SessionReader::open(&path)
            .unwrap()
            .scan(config.frame_bytes_max, |frame| {
                complete.push(frame);
                Ok(())
            })
            .unwrap();
        assert!(Session::from_frames(&complete).is_err(), "{mode}");
        drop(opened);
        super::commit_tests::clear(&path);
    }
}

#[test]
fn fork_keeps_chunk_backed_effects_and_unknown_intents_as_source_facts() {
    let input = "i🦀".repeat(14000);
    let output = "oλ".repeat(24000);
    for completed in [false, true] {
        let (path, mut opened, frames, outer) = frames(&input, &output);
        let input_id = frames.iter().flat_map(Frame::entries).next().unwrap().id;
        let output_id = frames.iter().flat_map(Frame::entries).last().unwrap().id;
        let Frame::Commit { checkpoint, .. } = &outer else {
            unreachable!()
        };
        for frame in &frames {
            if !completed
                && matches!(frame, Frame::NestedPayload { chunk } if chunk.phase == NestedPayloadPhase::Output)
            {
                break;
            }
            opened.writer.append_frame(frame).unwrap();
        }
        if completed {
            opened.writer.append_frame(&outer).unwrap();
        }
        let selected = if completed { output_id } else { input_id };
        let fork = path.with_extension("fork.smh");
        let second = path.with_extension("second.smh");
        let config = Config::default_valid();
        let limit = config.frame_bytes_max;
        let source_before = std::fs::read(&path).unwrap();
        let mut forked = SessionReader::open(&path)
            .unwrap()
            .fork_at(selected, &fork, config.clone())
            .unwrap();
        assert!(forked.writer.claim(checkpoint.submission_id).is_err());
        let reader = SessionReader::open(&fork).unwrap();
        reader.verify_full(limit).unwrap();
        reader.reject_pending_admissions(limit).unwrap();
        assert_eq!(
            reader.work_status(checkpoint.submission_id, limit).unwrap(),
            None
        );
        assert_eq!(reader.nested_payload(input_id, limit).unwrap(), input);
        if completed {
            assert_eq!(reader.nested_payload(output_id, limit).unwrap(), output);
        } else {
            assert!(reader.find_entry(output_id, limit).unwrap().is_none());
        }
        assert_eq!(reader.selected_tip(limit).unwrap(), Some(selected));
        let own = forked
            .writer
            .admit(
                &smith::message::Message::with_text(smith::message::Role::User, "new work"),
                None,
                None,
            )
            .unwrap();
        forked.writer.claim(own.submission_id).unwrap();
        forked
            .writer
            .settle(
                own.submission_id,
                Some(smith::stream::StopReason::EndTurn),
                None,
            )
            .unwrap();
        // A second fork of inherited history closes partial outer scopes without adopting either root.
        drop(reader.fork_at(selected, &second, config).unwrap());
        let twice = SessionReader::open(&second).unwrap();
        twice.verify_full(limit).unwrap();
        twice.reject_pending_admissions(limit).unwrap();
        assert_eq!(twice.nested_payload(input_id, limit).unwrap(), input);
        assert_eq!(twice.work_status(own.submission_id, limit).unwrap(), None);
        assert_eq!(std::fs::read(&path).unwrap(), source_before);
        if !completed {
            let output_start = frames.iter().position(|frame| matches!(frame, Frame::NestedPayload { chunk } if chunk.phase == NestedPayloadPhase::Output)).unwrap();
            for frame in &frames[output_start..] {
                opened.writer.append_frame(frame).unwrap();
            }
            opened.writer.append_frame(&outer).unwrap();
            assert!(
                SessionReader::open(&path)
                    .unwrap()
                    .find_entry(output_id, limit)
                    .unwrap()
                    .is_some()
            );
            assert!(
                SessionReader::open(&fork)
                    .unwrap()
                    .find_entry(output_id, limit)
                    .unwrap()
                    .is_none()
            );
        }
        drop((twice, reader, forked, opened));
        super::commit_tests::clear(&second);
        super::commit_tests::clear(&fork);
        super::commit_tests::clear(&path);
    }
}

#[test]
fn chunk_damage_owner_sequence_or_missing_bytes_never_publish_completion() {
    for mode in [
        "missing",
        "duplicate",
        "reorder",
        "root",
        "call",
        "effect",
        "index",
        "phase",
        "length",
        "utf8",
        "crc",
        "torn",
    ] {
        let text = "x".repeat(metadata::NESTED_CHUNK_BYTES_MAX + 7);
        let (path, mut opened, mut frames, outer) = frames(&text, &text);
        // First finish the input. Corrupt only the output phase.
        for frame in frames.drain(..3) {
            opened.writer.append_frame(&frame).unwrap();
        }
        let completion_id = frames.last().unwrap().entries()[0].id;
        match mode {
            "missing" => {
                frames.remove(0);
            }
            "duplicate" => {
                frames.insert(1, frames[0].clone());
            }
            "reorder" => frames.swap(0, 1),
            "root" | "call" | "effect" | "index" | "phase" | "length" | "utf8" => {
                let Frame::NestedPayload { chunk } = &mut frames[0] else {
                    unreachable!()
                };
                match mode {
                    "root" => chunk.submission_id = EntryId::new(),
                    "call" => chunk.call_id = smith::id::ToolCallId::new(),
                    "effect" => chunk.effect_id = EntryId::new(),
                    "index" => chunk.index = 1,
                    "phase" => chunk.phase = NestedPayloadPhase::Input,
                    "length" => {
                        chunk.bytes.pop();
                    }
                    _ => chunk.bytes[0] = 255,
                }
            }
            _ => {}
        }
        let limit = Config::default_valid().frame_bytes_max;
        // Writer rejects checksum-valid malformed links before durable acknowledgement.
        if !matches!(mode, "crc" | "torn") {
            let mut rejected = false;
            for frame in &frames {
                if opened.writer.append_frame(frame).is_err() {
                    rejected = true;
                    break;
                }
            }
            assert!(rejected, "{mode}");
            assert!(opened.writer.append_frame(&outer).is_err(), "{mode}");
        } else {
            let mut bytes = frames[0].encode(limit).unwrap();
            if mode == "crc" {
                bytes[crate::frame::HEADER_LEN + 4 + 53] ^= 1;
            } else {
                bytes.pop();
            }
            OpenOptions::new()
                .append(true)
                .open(&path)
                .unwrap()
                .write_all(&bytes)
                .unwrap();
        }
        drop(opened);
        let reader = SessionReader::open(&path).unwrap();
        assert!(
            reader.nested_payload(completion_id, limit).is_err(),
            "{mode}"
        );
        let Frame::Commit { checkpoint, .. } = outer else {
            unreachable!()
        };
        let status = reader.work_status(checkpoint.submission_id, limit);
        if mode != "crc" {
            assert_eq!(status.unwrap(), Some(WorkStatus::Running), "{mode}");
        }
        drop(reader);
        if let Ok(mut reopened) = open_session(&path, Config::default_valid()) {
            assert!(
                reopened.writer.claim(checkpoint.submission_id).is_err(),
                "{mode}"
            );
            assert!(
                reopened
                    .writer
                    .settle(checkpoint.submission_id, None, None)
                    .is_err(),
                "{mode}"
            );
        }
        super::commit_tests::clear(&path);
    }
}

#[test]
fn persisted_checksum_valid_bad_chunk_links_cannot_be_reopened_as_complete_output() {
    for mode in [
        "missing",
        "duplicate",
        "reorder",
        "root",
        "call",
        "effect",
        "index",
        "phase",
        "length",
        "utf8",
    ] {
        let text = "x".repeat(metadata::NESTED_CHUNK_BYTES_MAX + 7);
        let (path, opened, mut records, outer) = frames(&text, &text);
        let completion = records.last().unwrap().entries()[0].id;
        match mode {
            "missing" => {
                records.remove(3);
            }
            "duplicate" => {
                records.insert(4, records[3].clone());
            }
            "reorder" => records.swap(3, 4),
            _ => {
                let Frame::NestedPayload { chunk } = &mut records[3] else {
                    unreachable!()
                };
                match mode {
                    "root" => chunk.submission_id = EntryId::new(),
                    "call" => chunk.call_id = smith::id::ToolCallId::new(),
                    "effect" => chunk.effect_id = EntryId::new(),
                    "index" => chunk.index = 1,
                    "phase" => chunk.phase = NestedPayloadPhase::Input,
                    "length" => {
                        chunk.bytes.pop();
                    }
                    _ => chunk.bytes[0] = 255,
                }
            }
        }
        let limit = Config::default_valid().frame_bytes_max;
        let mut raw = OpenOptions::new().append(true).open(&path).unwrap();
        for record in records.iter().chain(std::iter::once(&outer)) {
            raw.write_all(&record.encode(limit).unwrap()).unwrap();
        }
        raw.sync_all().unwrap();
        drop(raw);
        drop(opened);
        let reader = SessionReader::open(&path).unwrap();
        assert!(reader.nested_payload(completion, limit).is_err(), "{mode}");
        assert!(reader.selected_session(limit).is_err(), "{mode}");
        drop(reader);
        assert!(
            open_session(&path, Config::default_valid()).is_err(),
            "{mode}"
        );
        super::commit_tests::clear(&path);
    }
}

#[test]
fn completed_payload_cannot_be_republished_by_a_flattened_marker_after_settlement() {
    let text = "x".repeat(metadata::NESTED_CHUNK_BYTES_MAX + 1);
    let (path, mut opened, records, outer) = frames(&text, &text);
    for record in &records {
        opened.writer.append_frame(record).unwrap();
    }
    opened.writer.append_frame(&outer).unwrap();
    let Frame::Commit { checkpoint, .. } = &outer else {
        unreachable!()
    };
    opened
        .writer
        .settle(checkpoint.submission_id, None, None)
        .unwrap();
    let mut entry = records.last().unwrap().entries()[0].clone();
    entry.id = EntryId::new();
    let flattened = Frame::Known { entry };
    assert!(opened.writer.append_frame(&flattened).is_err());
    let limit = Config::default_valid().frame_bytes_max;
    OpenOptions::new()
        .append(true)
        .open(&path)
        .unwrap()
        .write_all(&flattened.encode(limit).unwrap())
        .unwrap();
    drop(opened);
    assert!(
        SessionReader::open(&path)
            .unwrap()
            .selected_session(limit)
            .is_err()
    );
    super::commit_tests::clear(&path);
}

#[test]
fn payload_progress_has_fixed_layout_and_exact_total_and_count_bounds() {
    let root = EntryId::new();
    let call = smith::id::ToolCallId::new();
    let effect = EntryId::new();
    let mut ledger = WorkLedger {
        live: vec![(root, WorkStatus::Running)],
        tool: Some(call),
        ..WorkLedger::default()
    };
    let mut chunk = NestedPayloadChunk {
        submission_id: root,
        call_id: call,
        effect_id: effect,
        index: 0,
        phase: NestedPayloadPhase::Input,
        ordinal: 0,
        bytes: vec![b'x'; metadata::NESTED_CHUNK_BYTES_MAX],
    };
    for ordinal in 0..metadata::NESTED_CHUNKS_MAX {
        chunk.ordinal = ordinal.try_into().unwrap();
        ledger.observe_payload(&chunk).unwrap();
    }
    let progress = ledger.payload.as_ref().unwrap();
    assert_eq!(progress.bytes, metadata::NESTED_PAYLOAD_BYTES_MAX);
    assert_eq!(
        progress.bytes.div_ceil(metadata::NESTED_CHUNK_BYTES_MAX),
        metadata::NESTED_CHUNKS_MAX
    );
    #[cfg(target_arch = "x86_64")]
    assert_eq!(std::mem::size_of::<PayloadProgress>(), 64);
    chunk.ordinal += 1;
    assert!(ledger.observe_payload(&chunk).is_err());
}

#[test]
fn complete_chunks_without_their_publishing_marker_remain_unknown_after_reopen() {
    let text = "x".repeat(metadata::NESTED_CHUNK_BYTES_MAX + 1);
    for prefix in [2, 5] {
        let (path, mut opened, records, outer) = frames(&text, &text);
        for record in &records[..prefix] {
            opened.writer.append_frame(record).unwrap();
        }
        let missing = records[prefix].entries()[0].id;
        let Frame::Commit { checkpoint, .. } = &outer else {
            unreachable!()
        };
        assert!(opened.writer.append_frame(&outer).is_err());
        drop(opened);
        let limit = Config::default_valid().frame_bytes_max;
        let reader = SessionReader::open(&path).unwrap();
        assert_eq!(
            reader.work_status(checkpoint.submission_id, limit).unwrap(),
            Some(WorkStatus::Running)
        );
        assert_eq!(
            reader
                .selected_session(limit)
                .unwrap()
                .mutation_state(checkpoint.call_id)
                .unwrap(),
            MutationState::Unknown
        );
        assert!(reader.nested_payload(missing, limit).is_err());
        drop(reader);
        let mut reopened = open_session(&path, Config::default_valid()).unwrap();
        assert!(reopened.writer.claim(checkpoint.submission_id).is_err());
        assert!(
            reopened
                .writer
                .settle(checkpoint.submission_id, None, None)
                .is_err()
        );
        drop(reopened);
        super::commit_tests::clear(&path);
    }
}

#[test]
fn orphan_input_chunks_cannot_settle_or_acknowledge_an_effect_intent() {
    let text = "x".repeat(metadata::NESTED_CHUNK_BYTES_MAX + 1);
    let (path, mut opened, frames, outer) = frames(&text, &text);
    opened.writer.append_frame(&frames[0]).unwrap();
    let Frame::Commit { checkpoint, .. } = &outer else {
        unreachable!()
    };
    assert!(opened.writer.append_frame(&outer).is_err());
    assert!(
        opened
            .writer
            .settle(checkpoint.submission_id, None, None)
            .is_err()
    );
    assert!(opened.writer.append_frame(&frames[2]).is_err());
    drop(opened);
    super::commit_tests::clear(&path);
}