# Noko's speech engine on disk: the speak_to_me release it installs, updates # and launches. Every engine path, release URL and engine command line lives # here; Noko.qml only names the operations below and speaks JSON-RPC to `serve`. # # $XDG_DATA_HOME/nuguland/noko/ # engine//speak_to_me/ one unpacked release bundle per revision # engine/current -> the engine Noko runs # engine/previous -> kept until the current one has run once # engine/rejected revisions that failed their first run # models/ model cache, shared by every revision # # Every operation prints JSON lines on stdout, or one plain line on stderr and # exits 1. const release_default = "https://pkg.bugabinga.net/releases/speak_to_me" const revision_pattern = '^[0-9a-f]{40}$' def fail [message: string] { error make {msg: $message} } def emit [record: record] { print ($record | to json --raw) } def paths [] { let data = ($env.XDG_DATA_HOME? | default "") let base = ((if ($data | is-empty) { $env.HOME | path join .local share } else { $data }) | path join nuguland noko) if not ($base | str starts-with "/") { fail "XDG data path must be absolute" } let engine = ($base | path join engine) { base: $base engine: $engine models: ($base | path join models) current: ($engine | path join current) previous: ($engine | path join previous) rejected: ($engine | path join rejected) } } # The release directory; NOKO_RELEASE_URL points tests at a local server. def release [] { $env.NOKO_RELEASE_URL? | default $release_default | str trim --right --char "/" } def kind [path: string] { if ($path | path exists --no-symlink) { $path | path type } else { "missing" } } # The revision a link names, or null. def linked [link: string] { if (kind $link) != "symlink" { return null } let target = (^readlink $link | str trim) if ($target =~ $revision_pattern) { $target } else { null } } def bundle [revision: string] { (paths).engine | path join $revision speak_to_me } def executable [revision: string] { bundle $revision | path join bin speak_to_me } def rejected [] { let file = (paths).rejected if (kind $file) == "file" { open --raw $file | lines | where {|line| $line =~ $revision_pattern } } else { [] } } # Unpacked revisions besides current and previous: downloaded, not yet switched. def staged [] { let p = paths if (kind $p.engine) != "dir" { return [] } let keep = [(linked $p.current) (linked $p.previous)] ls --short-names $p.engine | where type == dir | get name | where {|name| $name =~ $revision_pattern and $name not-in $keep and ((executable $name) | path exists) } } # Atomic replacement of a symlink: a new link beside it, renamed over it. def link [target: string, at: string] { let temporary = $"($at).next" rm --force $temporary let made = (^ln -s $target $temporary | complete) if $made.exit_code != 0 { fail ($made.stderr | str trim) } let moved = (^mv -T -f $temporary $at | complete) if $moved.exit_code != 0 { rm --force $temporary fail ($moved.stderr | str trim) } } # The published manifest, checked field by field; the bundle URL must sit in # its own immutable revision directory of this release. def latest [] { let base = (release) let manifest = try { http get --raw --max-time 30sec $"($base)/latest/manifest.json" | decode utf-8 | from json } catch {|error| fail $"update check failed: ($error.msg)" } if not ($manifest | describe | str starts-with "record") { fail "update check failed: no manifest" } let revision = ($manifest.revision? | default "") let bundle = ($manifest.bundle? | default {}) let file = ($bundle.file? | default "") if ($manifest.schema? != 1) or ($manifest.name? != "speak_to_me") or ($revision !~ $revision_pattern) { fail "update check failed: unexpected manifest" } if ($file !~ '^[A-Za-z0-9._-]+\.tar\.zst$') or ($bundle.url? != $"($base)/($revision)/($file)") { fail "update check failed: unexpected bundle location" } let size = ($bundle.size? | default 0) let sha256 = ($bundle.sha256? | default "") if (($size | describe) != "int") or $size <= 0 or ($sha256 !~ '^[0-9a-f]{64}$') { fail "update check failed: unexpected bundle checksum" } {revision: $revision, url: $bundle.url, size: $size, sha256: $sha256} } def status [] { let p = paths emit { event: "status" installed: (linked $p.current) previous: (linked $p.previous) staged: (staged) typing: (which wtype | is-not-empty) } } def check [] { let p = paths let manifest = (latest) let installed = (linked $p.current) emit { event: "latest" revision: $manifest.revision size: $manifest.size installed: $installed staged: ($manifest.revision in (staged)) rejected: ($manifest.revision in (rejected)) } } # Download to a partial file, verify size and checksum, unpack beside the # installed revisions, and rename into place. Nothing installed is touched; a # failure leaves no partial file or half-unpacked directory behind. def install [] { let p = paths let manifest = (latest) let revision = $manifest.revision if $revision in (rejected) { fail "update failed its first run before" } let target = ($p.engine | path join $revision) if (kind $target) == "dir" and ((executable $revision) | path exists) { emit {event: "done", label: "ready", progress: 1, revision: $revision} return } mkdir $p.engine # Leftovers of an interrupted run, and staged revisions this one supersedes. let keep = [(linked $p.current) (linked $p.previous)] for entry in (ls --all --short-names $p.engine | get name) { let stale = ($entry | str starts-with ".download-") or ($entry | str starts-with ".unpack-") or ($entry =~ $revision_pattern and $entry not-in $keep) if $stale { rm --recursive --force ($p.engine | path join $entry) } } let partial = ($p.engine | path join $".download-($revision).tar.zst") let unpack = ($p.engine | path join $".unpack-($revision)") let failure = try { emit {event: "start", label: "downloading", progress: 0} let marker = ($p.engine | path join $".download-($revision).status") let url = $manifest.url let job = (job spawn { let outcome = try { http get --raw --max-time 30min $url | save --raw --force $partial "ok" } catch {|error| $error.msg } $outcome | save --force $marker }) mut shown = -1.0 while (job list | where id == $job | is-not-empty) { let received = (if ($partial | path exists) { ls $partial | get 0.size | into int } else { 0 }) let progress = ([($received / $manifest.size * 0.9) 0.9] | math min | math round --precision 2 | into float) if $progress != $shown { emit {event: "download", label: "downloading", progress: $progress} $shown = $progress } sleep 200ms } let outcome = (if ($marker | path exists) { open --raw $marker | str trim } else { "download stopped" }) rm --force $marker if $outcome != "ok" { fail $"download failed: ($outcome)" } emit {event: "verify", label: "verifying", progress: 0.9} let size = (ls $partial | get 0.size | into int) if $size != $manifest.size { fail "download did not match its size" } if (open --raw $partial | hash sha256) != $manifest.sha256 { fail "download did not match its checksum" } emit {event: "unpack", label: "unpacking", progress: 0.95} mkdir $unpack let unpacked = (^tar --zstd --extract --no-same-owner --file $partial --directory $unpack | complete) if $unpacked.exit_code != 0 { fail $"unpacking failed: ($unpacked.stderr | str trim)" } if (ls --all --short-names $unpack | get name) != ["speak_to_me"] { fail "bundle must hold one speak_to_me directory" } let inner = ($unpack | path join speak_to_me) let binary = ($inner | path join bin speak_to_me) if (kind $binary) != "file" { fail "bundle has no engine executable" } let shipped = try { open ($inner | path join manifest.json) } catch { {} } if ($shipped.revision? != $revision) { fail "bundle manifest names another revision" } let moved = (^mv -T $unpack $target | complete) if $moved.exit_code != 0 { fail ($moved.stderr | str trim) } null } catch {|error| $error.msg } rm --force $partial rm --recursive --force $unpack if $failure != null { fail $failure } emit {event: "done", label: "ready", progress: 1, revision: $revision} } # Point `current` at an unpacked revision; the replaced one stays as `previous`. def switch [revision: string] { let p = paths if ($revision !~ $revision_pattern) or not ((executable $revision) | path exists) { fail "no such engine revision" } let current = (linked $p.current) if $current == $revision { emit {event: "switched", revision: $revision, previous: (linked $p.previous)} return } if $current != null { link $current $p.previous } link $revision $p.current emit {event: "switched", revision: $revision, previous: $current} } # The current revision has run once: the previous one goes. def confirm [] { let p = paths let current = (linked $p.current) let previous = (linked $p.previous) if (kind $p.previous) != "missing" { rm --force $p.previous } if $previous != null and $previous != $current { rm --recursive --force ($p.engine | path join $previous) } emit {event: "confirmed", revision: $current} } # The current revision failed its first run: back to the previous one, and the # failed revision is never installed again. def rollback [] { let p = paths let current = (linked $p.current) let previous = (linked $p.previous) if $previous == null { fail "no previous engine" } link $previous $p.current rm --force $p.previous if $current != null and $current != $previous { $"($current)\n" | save --append $p.rejected rm --recursive --force ($p.engine | path join $current) } emit {event: "rolled-back", revision: $previous} } # Replace this process with the engine's JSON-RPC server on stdin/stdout. # The bundle is read-only: its own directory is the asset root, and models live # in the shared cache the data root names, so an update replaces neither. def serve [revision: string] { let p = paths if ($revision !~ $revision_pattern) { fail "no such engine revision" } let binary = (executable $revision) if not ($binary | path exists) { fail "no such engine revision" } mkdir $p.models exec $binary serve --data-root $p.models --assets-dir (bundle $revision | path join assets) } # `nu noko-engine.nu [revision]` def main [operation: string, revision?: string] { try { match $operation { "status" => (status) "check" => (check) "install" => (install) "switch" => (switch ($revision | default "")) "confirm" => (confirm) "rollback" => (rollback) "serve" => (serve ($revision | default "")) _ => (fail $"invalid operation: ($operation)") } } catch {|error| print --stderr $"noko-engine: ($error.msg)" exit 1 } }