Skip to content

Commit 427bda4

Browse files
k11kirkygreptile-apps[bot]
k11kirky
and
greptile-apps[bot]
authored
feat(LLM-Observability): Add new script to generate ai costs (PostHog#28071)
Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
1 parent f1adaf9 commit 427bda4

File tree

31 files changed

+974
-5493
lines changed

31 files changed

+974
-5493
lines changed

‎plugin-server/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@
3030
"services:clean": "cd .. && docker compose -f docker-compose.dev.yml rm -v",
3131
"services": "pnpm services:stop && pnpm services:clean && pnpm services:start",
3232
"build:cyclotron": "cd ../rust/cyclotron-node && pnpm run package",
33-
"pnpm:devPreinstall": "pnpm run build:cyclotron"
33+
"pnpm:devPreinstall": "pnpm run build:cyclotron",
34+
"update-ai-costs": "ts-node scripts/update-ai-costs.ts"
3435
},
3536
"graphile-worker": {
3637
"maxContiguousErrors": 300
@@ -131,6 +132,7 @@
131132
"babel-eslint": "^10.1.0",
132133
"c8": "^7.12.0",
133134
"deepmerge": "^4.2.2",
135+
"dotenv": "^16.4.7",
134136
"eslint": "^8.53.0",
135137
"eslint-config-prettier": "^9.1.0",
136138
"eslint-plugin-eslint-comments": "^3.2.0",

‎plugin-server/pnpm-lock.yaml‎

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎plugin-server/scripts/update-ai-costs.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
import dotenv from 'dotenv'
2+
import fs from 'fs'
3+
import path from 'path'
4+
5+
dotenv.config()
6+
7+
interface ModelRow {
8+
model: string
9+
cost: {
10+
prompt_token: number
11+
completion_token: number
12+
}
13+
}
14+
15+
const supportedProviderList = ['openai', 'anthropic', 'google', 'deepseek', 'perplexity', 'cohere', 'mistralai']
16+
17+
const main = async () => {
18+
if (!process.env.OPENROUTER_API_KEY) {
19+
console.error('OPENROUTER_API_KEY is not set')
20+
return
21+
}
22+
23+
const res = await fetch('https://openrouter.ai/api/v1/models', {
24+
headers: {
25+
Authorization: `Bearer ${process.env.OPENROUTER_API_KEY}`,
26+
},
27+
})
28+
if (!res.ok) {
29+
throw new Error(`Failed to fetch models: ${res.status} ${res.statusText}`)
30+
}
31+
let data
32+
try {
33+
data = await res.json()
34+
} catch (e) {
35+
throw new Error('Failed to parse API response as JSON')
36+
}
37+
console.log(data.data)
38+
const models = data.data
39+
40+
// Create main directory
41+
const baseDir = path.join(process.cwd(), 'src/utils/ai-costs')
42+
if (!fs.existsSync(baseDir)) {
43+
fs.mkdirSync(baseDir)
44+
}
45+
46+
// Group models by provider
47+
const providerModels = new Map<string, ModelRow[]>()
48+
49+
for (const model of models) {
50+
if (!model?.id || !model?.pricing?.prompt || !model?.pricing?.completion) {
51+
console.warn('Skipping invalid model:', model)
52+
continue
53+
}
54+
const [provider, ...modelParts] = model.id.split('/')
55+
if (!supportedProviderList.includes(provider)) {
56+
continue
57+
}
58+
if (!providerModels.has(provider)) {
59+
providerModels.set(provider, [])
60+
}
61+
62+
const modelRow: ModelRow = {
63+
model: modelParts.join('/'), // Only include the part after the provider
64+
cost: {
65+
prompt_token: parseFloat(model.pricing.prompt),
66+
completion_token: parseFloat(model.pricing.completion),
67+
},
68+
}
69+
70+
providerModels.get(provider)!.push(modelRow)
71+
}
72+
73+
// Generate files for each provider
74+
for (const [provider, models] of providerModels.entries()) {
75+
if (!fs.existsSync(baseDir)) {
76+
fs.mkdirSync(baseDir)
77+
}
78+
79+
const fileContent = `import type { ModelRow } from './types';\n\nexport const costs: ModelRow[] = ${JSON.stringify(
80+
models,
81+
null,
82+
2
83+
)};\n`
84+
fs.writeFileSync(path.join(baseDir, `${provider}.ts`), fileContent)
85+
}
86+
87+
// Create types.ts in the base directory
88+
const typesContent = `export interface ModelRow {
89+
model: string;
90+
cost: {
91+
prompt_token: number;
92+
completion_token: number;
93+
};
94+
}`
95+
fs.writeFileSync(path.join(baseDir, 'types.ts'), typesContent)
96+
}
97+
98+
;(async () => {
99+
await main()
100+
})().catch((e) => {
101+
console.error('Error updating AI costs:', e)
102+
process.exit(1)
103+
})

‎plugin-server/src/types.ts‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1317,61 +1317,6 @@ export type AppMetric2Type = {
13171317
count: number
13181318
}
13191319

1320-
interface TextOperator {
1321-
operator: 'equals' | 'startsWith' | 'includes'
1322-
value: string
1323-
}
1324-
1325-
export interface ModelDetails {
1326-
matches: string[]
1327-
searchTerms: string[]
1328-
info: {
1329-
releaseDate: string
1330-
maxTokens?: number
1331-
description: string
1332-
tradeOffs: string[]
1333-
benchmarks: {
1334-
[key: string]: number
1335-
}
1336-
capabilities: string[]
1337-
strengths: string[]
1338-
weaknesses: string[]
1339-
recommendations: string[]
1340-
}
1341-
}
1342-
1343-
export type ModelDetailsMap = {
1344-
[key: string]: ModelDetails
1345-
}
1346-
1347-
export interface ModelRow {
1348-
model: TextOperator
1349-
cost: {
1350-
prompt_token: number
1351-
completion_token: number
1352-
}
1353-
showInPlayground?: boolean
1354-
targetUrl?: string
1355-
dateRange?: {
1356-
start: string
1357-
end: string
1358-
}
1359-
}
1360-
1361-
export interface ModelRow {
1362-
model: TextOperator
1363-
cost: {
1364-
prompt_token: number
1365-
completion_token: number
1366-
}
1367-
showInPlayground?: boolean
1368-
targetUrl?: string
1369-
dateRange?: {
1370-
start: string
1371-
end: string
1372-
}
1373-
}
1374-
13751320
export interface CookielessConfig {
13761321
disabled: boolean
13771322
forceStatelessMode: boolean

0 commit comments

Comments
 (0)