package logs import ( "errors" "io" "os" "path/filepath" "strings" "github.com/klauspost/compress/zstd" ) type Store struct { DataDir string } type logEncoder interface { io.Writer Close() error } var newLogEncoder = func(w io.Writer) (logEncoder, error) { return zstd.NewWriter(w) } var copyLog = io.Copy var createTempLog = os.CreateTemp var renameLog = os.Rename var removeLog = os.Remove func (s Store) CreateLive(runID string) (*os.File, error) { path := filepath.Join(s.DataDir, "live", runID+".log") if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, err } return os.Create(path) } func (s Store) Finalize(runID string) (string, error) { live := filepath.Join(s.DataDir, "live", runID+".log") final := filepath.Join(s.DataDir, "logs", runID+".log.zst") if err := os.MkdirAll(filepath.Dir(final), 0o755); err != nil { return "", err } in, err := os.Open(live) if err != nil { return "", err } defer in.Close() out, err := createTempLog(filepath.Dir(final), ".luci-log-") if err != nil { return "", err } temporary := out.Name() keepTemporary := false defer func() { if !keepTemporary { _ = removeLog(temporary) } }() encoder, err := newLogEncoder(out) if err != nil { _ = out.Close() return "", err } _, copyErr := copyLog(encoder, in) encoderCloseErr := encoder.Close() syncErr := out.Sync() fileCloseErr := out.Close() if err := errors.Join(copyErr, encoderCloseErr, syncErr, fileCloseErr); err != nil { return "", err } if err := renameLog(temporary, final); err != nil { return "", err } keepTemporary = true if err := removeLog(live); err != nil { return "", err } return final, nil } type Tail struct { Text string Truncated bool } type combinedReadCloser struct { io.Reader decoder *zstd.Decoder file *os.File } func (c *combinedReadCloser) Close() error { c.decoder.Close() return c.file.Close() } func (s Store) Open(name string) (io.ReadCloser, error) { if name == "" || filepath.Base(name) != name { return nil, os.ErrNotExist } live, err := os.Open(filepath.Join(s.DataDir, "live", name+".log")) if err == nil { return live, nil } if !os.IsNotExist(err) { return nil, err } file, err := os.Open(filepath.Join(s.DataDir, "logs", name+".log.zst")) if err != nil { return nil, err } decoder, err := zstd.NewReader(file) if err != nil { file.Close() return nil, err } return &combinedReadCloser{Reader: decoder, decoder: decoder, file: file}, nil } func (s Store) Tail(name string, limit int) (Tail, error) { reader, err := s.Open(name) if err != nil { return Tail{}, err } defer reader.Close() if limit < 1 { return Tail{}, nil } buffer := &tailBuffer{ring: make([]byte, limit)} sanitizer := NewSanitizer(buffer, nil) _, copyErr := io.Copy(sanitizer, reader) closeErr := sanitizer.Close() if err := errors.Join(copyErr, closeErr); err != nil { return Tail{}, err } data := buffer.bytes() if buffer.total > limit { for len(data) > 0 && data[0]&0xc0 == 0x80 { data = data[1:] } } return Tail{Text: strings.ToValidUTF8(string(data), "�"), Truncated: buffer.total > limit}, nil } type tailBuffer struct { ring []byte total, pos int } func (b *tailBuffer) Write(data []byte) (int, error) { for _, value := range data { b.ring[b.pos%len(b.ring)] = value b.pos++ } b.total += len(data) return len(data), nil } func (b *tailBuffer) bytes() []byte { count := min(b.total, len(b.ring)) data := make([]byte, count) start := 0 if b.total > len(b.ring) { start = b.pos % len(b.ring) } for i := range data { data[i] = b.ring[(start+i)%len(b.ring)] } return data } func (s Store) Spool(name string) (*os.File, error) { reader, err := s.Open(name) if err != nil { return nil, err } tmp, err := os.CreateTemp("", "luci-log-*") if err != nil { reader.Close() return nil, err } _ = os.Remove(tmp.Name()) sanitizer := NewSanitizer(tmp, nil) _, copyErr := io.Copy(sanitizer, reader) sanitizeErr := sanitizer.Close() closeErr := reader.Close() if err := errors.Join(copyErr, sanitizeErr, closeErr); err != nil { tmp.Close() return nil, err } if _, err := tmp.Seek(0, io.SeekStart); err != nil { tmp.Close() return nil, err } return tmp, nil }