Skip to content

Commit 8c32b96

Browse files
k11kirkygreptile-apps[bot]
k11kirky
and
greptile-apps[bot]
authored
feat(llm-o): connect Session replay to LLM Observability (PostHog#29224)
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
1 parent f5ce657 commit 8c32b96

File tree

9 files changed

+203
-18
lines changed

9 files changed

+203
-18
lines changed

‎frontend/src/scenes/activity/explore/EventDetails.tsx‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { HTMLElementsDisplay } from 'lib/components/HTMLElementsDisplay/HTMLElem
55
import { JSONViewer } from 'lib/components/JSONViewer'
66
import { PropertiesTable } from 'lib/components/PropertiesTable'
77
import { dayjs } from 'lib/dayjs'
8+
import { IconOpenInNew } from 'lib/lemon-ui/icons'
89
import { LemonButton } from 'lib/lemon-ui/LemonButton'
910
import { LemonTableProps } from 'lib/lemon-ui/LemonTable'
1011
import { LemonTabs } from 'lib/lemon-ui/LemonTabs'
@@ -14,6 +15,7 @@ import { pluralize } from 'lib/utils'
1415
import { AutocaptureImageTab, autocaptureToImage } from 'lib/utils/event-property-utls'
1516
import { ConversationDisplay } from 'products/llm_observability/frontend/ConversationDisplay/ConversationDisplay'
1617
import { useState } from 'react'
18+
import { urls } from 'scenes/urls'
1719

1820
import { EventType, PropertyDefinitionType } from '~/types'
1921

@@ -145,6 +147,17 @@ export function EventDetails({ event, tableProps }: EventDetailsProps): JSX.Elem
145147
label: 'Conversation',
146148
content: (
147149
<div className="mx-3 -mt-2 mb-2 space-y-2">
150+
{event.properties.$session_id ? (
151+
<div className="flex flex-row items-center gap-2">
152+
<Link
153+
to={urls.replay(undefined, undefined, event.properties.$session_id)}
154+
className="flex flex-row gap-1 items-center"
155+
>
156+
<IconOpenInNew />
157+
<span>View session recording</span>
158+
</Link>
159+
</div>
160+
) : null}
148161
<ConversationDisplay eventProperties={event.properties} />
149162
</div>
150163
),

‎frontend/src/scenes/session-recordings/player/inspector/components/AIEventItems.tsx‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import { JSONViewer } from 'lib/components/JSONViewer'
2+
import { IconExclamation } from 'lib/lemon-ui/icons'
3+
import { isObject } from 'lib/utils'
4+
import { cn } from 'lib/utils/css-classes'
5+
import { ConversationMessagesDisplay } from 'products/llm_observability/frontend/ConversationDisplay/ConversationMessagesDisplay'
6+
import { LLMInputOutput } from 'products/llm_observability/frontend/LLMInputOutput'
7+
8+
export function AIEventExpanded({ event }: { event: Record<string, any> }): JSX.Element {
9+
let input = event.properties.$ai_input_state
10+
let output = event.properties.$ai_output_state ?? event.properties.$ai_error
11+
let raisedError = event.properties.$ai_is_error
12+
if (event.event === '$ai_generation') {
13+
input = event.properties.$ai_input
14+
output = event.properties.$ai_output_choices ?? event.properties.$ai_output
15+
raisedError = event.properties.$ai_is_error
16+
}
17+
return (
18+
<div>
19+
{event.event === '$ai_generation' ? (
20+
<ConversationMessagesDisplay
21+
input={event.properties.$ai_input}
22+
output={
23+
event.properties.$ai_is_error
24+
? event.properties.$ai_error
25+
: event.properties.$ai_output_choices ?? event.properties.$ai_output
26+
}
27+
httpStatus={event.properties.$ai_http_status}
28+
raisedError={event.properties.$ai_is_error}
29+
/>
30+
) : (
31+
<LLMInputOutput
32+
inputDisplay={
33+
<div className="p-2 text-xs border rounded bg-[var(--bg-fill-secondary)]">
34+
{isObject(input) ? (
35+
<JSONViewer src={input} collapsed={2} />
36+
) : (
37+
<span className="font-mono">{JSON.stringify(input ?? null)}</span>
38+
)}
39+
</div>
40+
}
41+
outputDisplay={
42+
<div
43+
className={cn(
44+
'p-2 text-xs border rounded',
45+
!raisedError
46+
? 'bg-[var(--bg-fill-success-tertiary)]'
47+
: 'bg-[var(--bg-fill-error-tertiary)]'
48+
)}
49+
>
50+
{isObject(output) ? (
51+
<JSONViewer src={output} collapsed={2} />
52+
) : (
53+
<span className="font-mono">{JSON.stringify(output ?? null)}</span>
54+
)}
55+
</div>
56+
}
57+
/>
58+
)}
59+
</div>
60+
)
61+
}
62+
63+
export function AIEventSummary({ event }: { event: Record<string, any> }): JSX.Element | null {
64+
if (event.properties.$ai_is_error) {
65+
return (
66+
<div className="flex items-center gap-1 text-danger">
67+
<IconExclamation />
68+
<span>Error</span>
69+
</div>
70+
)
71+
}
72+
return null
73+
}

‎frontend/src/scenes/session-recordings/player/inspector/components/ItemEvent.tsx‎

Lines changed: 60 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { insightUrlForEvent } from 'scenes/insights/utils'
1717
import { eventPropertyFilteringLogic } from 'scenes/session-recordings/player/inspector/components/eventPropertyFilteringLogic'
1818

1919
import { InspectorListItemEvent } from '../playerInspectorLogic'
20+
import { AIEventExpanded, AIEventSummary } from './AIEventItems'
2021
import { SimpleKeyValueList } from './SimpleKeyValueList'
2122

2223
export interface ItemEventProps {
@@ -65,6 +66,10 @@ export function ItemEvent({ item }: ItemEventProps): JSX.Element {
6566
<SummarizeWebVitals properties={item.data.properties} />
6667
) : item.data.elements.length ? (
6768
<AutocapturePreviewImage elements={item.data.elements} />
69+
) : item.data.event === '$ai_generation' ||
70+
item.data.event === '$ai_span' ||
71+
item.data.event === '$ai_trace' ? (
72+
<AIEventSummary event={item.data} />
6873
) : null
6974

7075
return (
@@ -92,9 +97,20 @@ export function ItemEvent({ item }: ItemEventProps): JSX.Element {
9297
}
9398

9499
export function ItemEventDetail({ item }: ItemEventProps): JSX.Element {
100+
// // Check if this is an LLM-related event
101+
const isAIEvent =
102+
item.data.event === '$ai_generation' || item.data.event === '$ai_span' || item.data.event === '$ai_trace'
103+
95104
const [activeTab, setActiveTab] = useState<
96-
'properties' | 'flags' | 'image' | 'elements' | '$set_properties' | '$set_once_properties' | 'raw'
97-
>('properties')
105+
| 'properties'
106+
| 'flags'
107+
| 'image'
108+
| 'elements'
109+
| '$set_properties'
110+
| '$set_once_properties'
111+
| 'raw'
112+
| 'conversation'
113+
>(isAIEvent ? 'conversation' : 'properties')
98114

99115
const insightUrl = insightUrlForEvent(item.data)
100116
const { filterProperties } = useValues(eventPropertyFilteringLogic)
@@ -120,22 +136,44 @@ export function ItemEventDetail({ item }: ItemEventProps): JSX.Element {
120136
}
121137
}
122138

139+
// Get trace ID for linking to LLM trace view
140+
const traceId = item.data.properties.$ai_trace_id
141+
const traceUrl = traceId
142+
? `/llm-observability/traces/${traceId}${
143+
item.data.id && item.data.event !== '$ai_trace' ? `?event=${item.data.id}` : ''
144+
}`
145+
: null
146+
123147
return (
124148
<div data-attr="item-event" className="font-light w-full">
125149
<div className="px-2 py-1 text-xs border-t">
126-
{insightUrl ? (
150+
{insightUrl || traceUrl ? (
127151
<>
128-
<div className="flex justify-end">
129-
<LemonButton
130-
size="xsmall"
131-
type="secondary"
132-
sideIcon={<IconOpenInNew />}
133-
data-attr="recordings-event-to-insights"
134-
to={insightUrl}
135-
targetBlank
136-
>
137-
Try out in Insights
138-
</LemonButton>
152+
<div className="flex justify-end gap-2">
153+
{insightUrl && (
154+
<LemonButton
155+
size="xsmall"
156+
type="secondary"
157+
sideIcon={<IconOpenInNew />}
158+
data-attr="recordings-event-to-insights"
159+
to={insightUrl}
160+
targetBlank
161+
>
162+
Try out in Insights
163+
</LemonButton>
164+
)}
165+
{traceUrl && (
166+
<LemonButton
167+
size="xsmall"
168+
type="secondary"
169+
sideIcon={<IconOpenInNew />}
170+
data-attr="recordings-event-to-llm-trace"
171+
to={traceUrl}
172+
targetBlank
173+
>
174+
View LLM Trace
175+
</LemonButton>
176+
)}
139177
</div>
140178
<LemonDivider dashed />
141179
</>
@@ -187,6 +225,14 @@ export function ItemEventDetail({ item }: ItemEventProps): JSX.Element {
187225
content: <AutocaptureImageTab elements={item.data.elements} />,
188226
}
189227
: null,
228+
// Add conversation tab for $ai_generation events
229+
isAIEvent
230+
? {
231+
key: 'conversation',
232+
label: 'Conversation',
233+
content: <AIEventExpanded event={item.data} />,
234+
}
235+
: null,
190236
Object.keys(setProperties).length > 0
191237
? {
192238
key: '$set_properties',

‎frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -890,7 +890,12 @@ export const sessionRecordingDataLogic = kea<sessionRecordingDataLogicType>([
890890
(sessionEventsData): RecordingEventType[] =>
891891
(sessionEventsData || []).filter((e) => e.event === '$web_vitals'),
892892
],
893-
893+
AIEvents: [
894+
(s) => [s.sessionEventsData],
895+
(sessionEventsData): RecordingEventType[] =>
896+
// see if event start with $ai_
897+
(sessionEventsData || []).filter((e) => e.event.startsWith('$ai_')),
898+
],
894899
windowIdForTimestamp: [
895900
(s) => [s.segments],
896901
(segments) =>
@@ -1168,6 +1173,12 @@ export const sessionRecordingDataLogic = kea<sessionRecordingDataLogicType>([
11681173
actions.loadFullEventData(value)
11691174
}
11701175
},
1176+
AIEvents: (value: RecordingEventType[]) => {
1177+
// we preload all AI data, so it can be used before user interaction
1178+
if (value.length > 0) {
1179+
actions.loadFullEventData(value)
1180+
}
1181+
},
11711182
isRecentAndInvalid: (prev: boolean, next: boolean) => {
11721183
if (!prev && next) {
11731184
posthog.capture('recording cannot playback yet', {

‎plugin-server/src/utils/ai-costs/anthropic.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ export const costs: ModelRow[] = [
1515
completion_token: 0.000015,
1616
},
1717
},
18+
{
19+
model: 'claude-3.7-sonnet:thinking',
20+
cost: {
21+
prompt_token: 0.000003,
22+
completion_token: 0.000015,
23+
},
24+
},
1825
{
1926
model: 'claude-3.5-haiku-20241022:beta',
2027
cost: {

‎plugin-server/src/utils/ai-costs/deepseek.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ export const costs: ModelRow[] = [
6767
{
6868
model: 'deepseek-chat',
6969
cost: {
70-
prompt_token: 0.0000009,
71-
completion_token: 0.0000009,
70+
prompt_token: 0.00000125,
71+
completion_token: 0.00000125,
7272
},
7373
},
7474
{

‎plugin-server/src/utils/ai-costs/google.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
import type { ModelRow } from './types'
22

33
export const costs: ModelRow[] = [
4+
{
5+
model: 'gemini-2.0-flash-lite-001',
6+
cost: {
7+
prompt_token: 0.000000075,
8+
completion_token: 0.0000003,
9+
},
10+
},
411
{
512
model: 'gemini-2.0-flash-001',
613
cost: {

‎products/llm_observability/frontend/LLMObservabilityTraceScene.tsx‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import clsx from 'clsx'
55
import { BindLogic, useValues } from 'kea'
66
import { JSONViewer } from 'lib/components/JSONViewer'
77
import { NotFound } from 'lib/components/NotFound'
8-
import { IconArrowDown, IconArrowUp } from 'lib/lemon-ui/icons'
8+
import { IconArrowDown, IconArrowUp, IconOpenInNew } from 'lib/lemon-ui/icons'
99
import { identifierToHuman, isObject, pluralize } from 'lib/utils'
1010
import { cn } from 'lib/utils/css-classes'
1111
import React, { useEffect, useRef, useState } from 'react'
@@ -29,6 +29,8 @@ import {
2929
formatLLMEventTitle,
3030
formatLLMLatency,
3131
formatLLMUsage,
32+
getSessionID,
33+
hasSessionID,
3234
isLLMTraceEvent,
3335
removeMilliseconds,
3436
} from './utils'
@@ -384,6 +386,17 @@ const EventContent = React.memo(({ event }: { event: LLMTrace | LLMTraceEvent |
384386
/>
385387
)}
386388
{isLLMTraceEvent(event) && <ParametersHeader eventProperties={event.properties} />}
389+
{hasSessionID(event) && (
390+
<div className="flex flex-row items-center gap-2">
391+
<Link
392+
to={urls.replay(undefined, undefined, getSessionID(event) ?? '')}
393+
className="flex flex-row gap-1 items-center"
394+
>
395+
<IconOpenInNew />
396+
<span>View session recording</span>
397+
</Link>
398+
</div>
399+
)}
387400
</header>
388401
{isLLMTraceEvent(event) ? (
389402
event.event === '$ai_generation' ? (

‎products/llm_observability/frontend/utils.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,21 @@ export function isLLMTraceEvent(item: LLMTrace | LLMTraceEvent): item is LLMTrac
4747
return 'properties' in item
4848
}
4949

50+
export function hasSessionID(event: LLMTrace | LLMTraceEvent): boolean {
51+
if (isLLMTraceEvent(event)) {
52+
return 'properties' in event && typeof event.properties.$session_id === 'string'
53+
}
54+
return '$session_id' in event
55+
}
56+
57+
export function getSessionID(event: LLMTrace | LLMTraceEvent): string | null {
58+
if (isLLMTraceEvent(event)) {
59+
return event.properties.$session_id || null
60+
}
61+
62+
return event.events.find((e) => e.properties.$session_id !== null)?.properties.$session_id || null
63+
}
64+
5065
export function isOpenAICompatToolCall(input: unknown): input is OpenAIToolCall {
5166
return (
5267
input !== null &&

0 commit comments

Comments
 (0)