Skip to content

Commit 42d0e0e

Browse files
richiejprichiejp
committed
fix(tracing): Show full LLM opts and deltas
Signed-off-by: Richard Palethorpe <io@richiejp.com>
1 parent ae73f45 commit 42d0e0e

File tree

2 files changed

+71
-35
lines changed

2 files changed

+71
-35
lines changed

‎core/backend/llm.go‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
8484
}
8585

8686
// in GRPC, the backend is supposed to answer to 1 single token if stream is not supported
87+
var capturedPredictOpts *proto.PredictOptions
8788
fn := func() (LLMResponse, error) {
8889
opts := gRPCPredictOpts(*c, loader.ModelPath)
8990
// Merge request-level metadata (overrides config defaults)
@@ -111,6 +112,7 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
111112
opts.LogitBias = string(logitBiasJSON)
112113
}
113114
}
115+
capturedPredictOpts = opts
114116

115117
tokenUsage := TokenUsage{}
116118

@@ -245,29 +247,19 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
245247
trace.InitBackendTracingIfEnabled(o.TracingMaxItems)
246248

247249
traceData := map[string]any{
248-
"prompt": s,
249-
"use_tokenizer_template": c.TemplateConfig.UseTokenizerTemplate,
250-
"chat_template": c.TemplateConfig.Chat,
251-
"function_template": c.TemplateConfig.Functions,
252-
"grammar": c.Grammar,
253-
"stop_words": c.StopWords,
254-
"streaming": tokenCallback != nil,
255-
"images_count": len(images),
256-
"videos_count": len(videos),
257-
"audios_count": len(audios),
250+
"chat_template": c.TemplateConfig.Chat,
251+
"function_template": c.TemplateConfig.Functions,
252+
"streaming": tokenCallback != nil,
253+
"images_count": len(images),
254+
"videos_count": len(videos),
255+
"audios_count": len(audios),
258256
}
259257

260258
if len(messages) > 0 {
261259
if msgJSON, err := json.Marshal(messages); err == nil {
262260
traceData["messages"] = string(msgJSON)
263261
}
264262
}
265-
if tools != "" {
266-
traceData["tools"] = tools
267-
}
268-
if toolChoice != "" {
269-
traceData["tool_choice"] = toolChoice
270-
}
271263
if reasoningJSON, err := json.Marshal(c.ReasoningConfig); err == nil {
272264
traceData["reasoning_config"] = string(reasoningJSON)
273265
}
@@ -277,15 +269,6 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
277269
"mixed_mode": c.FunctionsConfig.GrammarConfig.MixedMode,
278270
"xml_format_preset": c.FunctionsConfig.XMLFormatPreset,
279271
}
280-
if c.Temperature != nil {
281-
traceData["temperature"] = *c.Temperature
282-
}
283-
if c.TopP != nil {
284-
traceData["top_p"] = *c.TopP
285-
}
286-
if c.Maxtokens != nil {
287-
traceData["max_tokens"] = *c.Maxtokens
288-
}
289272

290273
startTime := time.Now()
291274
originalFn := fn
@@ -299,6 +282,42 @@ func ModelInference(ctx context.Context, s string, messages schema.Messages, ima
299282
"completion": resp.Usage.Completion,
300283
}
301284

285+
if len(resp.ChatDeltas) > 0 {
286+
chatDeltasInfo := map[string]any{
287+
"total_deltas": len(resp.ChatDeltas),
288+
}
289+
var contentParts, reasoningParts []string
290+
toolCallCount := 0
291+
for _, d := range resp.ChatDeltas {
292+
if d.Content != "" {
293+
contentParts = append(contentParts, d.Content)
294+
}
295+
if d.ReasoningContent != "" {
296+
reasoningParts = append(reasoningParts, d.ReasoningContent)
297+
}
298+
toolCallCount += len(d.ToolCalls)
299+
}
300+
if len(contentParts) > 0 {
301+
chatDeltasInfo["content"] = strings.Join(contentParts, "")
302+
}
303+
if len(reasoningParts) > 0 {
304+
chatDeltasInfo["reasoning_content"] = strings.Join(reasoningParts, "")
305+
}
306+
if toolCallCount > 0 {
307+
chatDeltasInfo["tool_call_count"] = toolCallCount
308+
}
309+
traceData["chat_deltas"] = chatDeltasInfo
310+
}
311+
312+
if capturedPredictOpts != nil {
313+
if optsJSON, err := json.Marshal(capturedPredictOpts); err == nil {
314+
var optsMap map[string]any
315+
if err := json.Unmarshal(optsJSON, &optsMap); err == nil {
316+
traceData["predict_options"] = optsMap
317+
}
318+
}
319+
}
320+
302321
errStr := ""
303322
if err != nil {
304323
errStr = err.Error()

‎core/http/react-ui/src/pages/Traces.jsx‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,17 @@ function AudioSnippet({ data }) {
113113
)
114114
}
115115

116-
// Expandable data fields for backend traces
117-
function DataFields({ data }) {
116+
function isPlainObject(value) {
117+
return value !== null && typeof value === 'object' && !Array.isArray(value)
118+
}
119+
120+
function fieldSummary(value) {
121+
const count = Object.keys(value).length
122+
return `{${count} field${count !== 1 ? 's' : ''}}`
123+
}
124+
125+
// Expandable data fields for backend traces (recursive for nested objects)
126+
function DataFields({ data, nested }) {
118127
const [expandedFields, setExpandedFields] = useState({})
119128
const filtered = Object.entries(data).filter(([key]) => !AUDIO_DATA_KEYS.has(key))
120129
if (filtered.length === 0) return null
@@ -125,32 +134,40 @@ function DataFields({ data }) {
125134

126135
return (
127136
<div>
128-
<h4 style={{ fontSize: '0.8125rem', fontWeight: 600, marginBottom: 'var(--spacing-xs)' }}>Data Fields</h4>
137+
{!nested && <h4 style={{ fontSize: '0.8125rem', fontWeight: 600, marginBottom: 'var(--spacing-xs)' }}>Data Fields</h4>}
129138
<div style={{ border: '1px solid var(--color-border)', borderRadius: 'var(--radius-md)', overflow: 'hidden' }}>
130139
{filtered.map(([key, value]) => {
131-
const large = isLargeValue(value)
140+
const objValue = isPlainObject(value)
141+
const large = !objValue && isLargeValue(value)
142+
const expandable = objValue || large
132143
const expanded = expandedFields[key]
133144
return (
134145
<div key={key} style={{ borderBottom: '1px solid var(--color-border)' }}>
135146
<div
136-
onClick={large ? () => toggleField(key) : undefined}
147+
onClick={expandable ? () => toggleField(key) : undefined}
137148
style={{
138149
display: 'flex', alignItems: 'center', gap: 'var(--spacing-xs)',
139150
padding: 'var(--spacing-xs) var(--spacing-sm)',
140-
cursor: large ? 'pointer' : 'default',
151+
cursor: expandable ? 'pointer' : 'default',
141152
fontSize: '0.8125rem',
142153
}}
143154
>
144-
{large ? (
155+
{expandable ? (
145156
<i className={`fas fa-chevron-${expanded ? 'down' : 'right'}`} style={{ fontSize: '0.6rem', color: 'var(--color-text-secondary)', width: 12, flexShrink: 0 }} />
146157
) : (
147158
<span style={{ width: 12, flexShrink: 0 }} />
148159
)}
149160
<span style={{ fontFamily: 'monospace', color: 'var(--color-primary)', flexShrink: 0 }}>{key}</span>
150-
{!large && <span style={{ fontFamily: 'monospace', fontSize: '0.75rem', color: 'var(--color-text-secondary)' }}>{formatValue(value)}</span>}
151-
{large && !expanded && <span style={{ fontSize: '0.75rem', color: 'var(--color-text-secondary)', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{truncateValue(value, 120)}</span>}
161+
{objValue && !expanded && <span style={{ fontSize: '0.75rem', color: 'var(--color-text-secondary)' }}>{fieldSummary(value)}</span>}
162+
{!objValue && !large && <span style={{ fontFamily: 'monospace', fontSize: '0.75rem', color: 'var(--color-text-secondary)' }}>{formatValue(value)}</span>}
163+
{!objValue && large && !expanded && <span style={{ fontSize: '0.75rem', color: 'var(--color-text-secondary)', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{truncateValue(value, 120)}</span>}
152164
</div>
153-
{large && expanded && (
165+
{expanded && objValue && (
166+
<div style={{ padding: '0 0 var(--spacing-xs) var(--spacing-md)' }}>
167+
<DataFields data={value} nested />
168+
</div>
169+
)}
170+
{expanded && large && (
154171
<div style={{ padding: '0 var(--spacing-sm) var(--spacing-sm)' }}>
155172
<pre style={{
156173
background: 'var(--color-bg-primary)', border: '1px solid var(--color-border)',

0 commit comments

Comments
 (0)