From 7dd0067b56caba6df1bd8719c500d97b8fe1ff78 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 25 Jun 2026 14:41:28 +0200 Subject: [PATCH 1/6] WIP feat: Add new prompts API --- js/src/experimental-prompt-api.test.ts | 339 ++++++++++ js/src/experimental-prompt-api.ts | 816 +++++++++++++++++++++++++ js/src/exports.ts | 2 + 3 files changed, 1157 insertions(+) create mode 100644 js/src/experimental-prompt-api.test.ts create mode 100644 js/src/experimental-prompt-api.ts diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts new file mode 100644 index 000000000..e2d5f0987 --- /dev/null +++ b/js/src/experimental-prompt-api.test.ts @@ -0,0 +1,339 @@ +import { describe, expect, test } from "vitest"; +import { + prompt, + s, + type InputOf, + type OutputOf, +} from "./experimental-prompt-api"; + +describe("experimental prompt API", () => { + test("builds typed prompts and translates to OpenAI chat args", () => { + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + input: s.object({ + ticket: s.string(), + }), + output: s.object({ + subject: s.string(), + body: s.string(), + urgency: s.enum(["low", "medium", "high"]), + }), + render: ({ input }) => [ + prompt.system`You write concise support replies.`, + prompt.user`Ticket: ${input.ticket}`, + ], + }); + + type SupportReplyInput = InputOf; + type SupportReplyOutput = OutputOf; + + const input: SupportReplyInput = { ticket: "I cannot find eval history." }; + const output: SupportReplyOutput = { + subject: "Finding eval history", + body: "Here is where to look.", + urgency: "low", + }; + + const built = supportReply.build(input); + expect(built.parseOutput(output)).toEqual(output); + expect(built.to(prompt.adapters.openAIChat)).toMatchObject({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You write concise support replies." }, + { role: "user", content: "Ticket: I cannot find eval history." }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "support-reply_output", + strict: true, + schema: { + type: "object", + required: ["subject", "body", "urgency"], + }, + }, + }, + span_info: { + metadata: { + prompt: { + root: { slug: "support-reply" }, + prompts: [ + { + slug: "support-reply", + role: "root", + input, + }, + ], + }, + }, + }, + }); + }); + + test("includes nested prompts and records prompt dependencies", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + version: "v3", + input: s.object({ + company: s.string(), + }), + render: ({ input }) => [ + prompt.system`Use ${input.company}'s voice. Be warm and direct.`, + ], + }); + + const supportReply = prompt.define({ + slug: "support-reply", + version: "v8", + input: s.object({ + company: s.string(), + ticket: s.string(), + }), + render: ({ input, include }) => [ + include(brandVoice, { company: input.company }), + prompt.user`Draft a reply for: ${input.ticket}`, + ], + }); + + const built = supportReply.build({ + company: "Braintrust", + ticket: "Where did my experiment go?", + }); + + expect(built.messages).toEqual([ + { + role: "system", + content: "Use Braintrust's voice. Be warm and direct.", + }, + { + role: "user", + content: "Draft a reply for: Where did my experiment go?", + }, + ]); + expect(built.dependencies.prompts).toMatchObject([ + { + slug: "support-reply", + version: "v8", + role: "root", + }, + { + slug: "brand-voice", + version: "v3", + role: "include", + parent: "support-reply", + input: { company: "Braintrust" }, + }, + ]); + }); + + test("builds schema prompt fields from merged parent input and overrides", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + version: "v3", + input: s.object({ + company: s.string(), + tone: s.string(), + }), + render: ({ input }) => [ + prompt.system`Use ${input.company}'s ${input.tone} voice.`, + ], + }); + + const supportReply = prompt.define({ + slug: "support-reply", + version: "v8", + input: s.object({ + company: s.string(), + ticket: s.string(), + voice: s.prompt(brandVoice), + }), + render: ({ input }) => [ + input.voice, + prompt.user`Draft a reply for: ${input.ticket}`, + ], + }); + + type SupportReplyInput = InputOf; + const input: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { tone: "direct" }, + }; + + const built = supportReply.build(input); + + expect(built.messages).toEqual([ + { + role: "system", + content: "Use Braintrust's direct voice.", + }, + { + role: "user", + content: "Draft a reply for: Where did my experiment go?", + }, + ]); + expect(built.dependencies.prompts).toMatchObject([ + { + slug: "support-reply", + role: "root", + input: { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { + type: "built_prompt", + root: { slug: "brand-voice", version: "v3" }, + }, + }, + }, + { + slug: "brand-voice", + version: "v3", + role: "include", + parent: "support-reply", + input: { + company: "Braintrust", + tone: "direct", + }, + }, + ]); + }); + + test("allows dynamic s.prompt fields that receive a prompt definition", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + input: s.object({ + company: s.string(), + tone: s.string(), + }), + render: ({ input }) => [ + prompt.system`Use ${input.company}'s ${input.tone} voice.`, + ], + }); + + const supportReply = prompt.define({ + slug: "support-reply", + input: s.object({ + company: s.string(), + ticket: s.string(), + voice: s.prompt(), + }), + render: ({ input }) => [ + input.voice, + prompt.user`Draft a reply for: ${input.ticket}`, + ], + }); + + const built = supportReply.build({ + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { + prompt: brandVoice, + input: { tone: "calm" }, + }, + }); + + expect(built.messages).toEqual([ + { + role: "system", + content: "Use Braintrust's calm voice.", + }, + { + role: "user", + content: "Draft a reply for: Where did my experiment go?", + }, + ]); + }); + + test("accepts a built prompt as input and sanitizes dependencies", () => { + const summarizeTicket = prompt.define({ + slug: "summarize-ticket", + input: s.object({ + ticket: s.string(), + }), + render: ({ input }) => [prompt.user`Summarize: ${input.ticket}`], + }); + + const critiquePrompt = prompt.define({ + slug: "critique-prompt", + input: s.object({ + prior: s.builtPrompt(), + }), + render: ({ input }) => [ + prompt.user`Critique this prompt:\n${prompt.asText(input.prior)}`, + ], + }); + + const prior = summarizeTicket.build({ ticket: "Billing looks wrong." }); + const built = critiquePrompt.build({ prior }); + + expect(built.messages).toEqual([ + { + role: "user", + content: "Critique this prompt:\nuser: Summarize: Billing looks wrong.", + }, + ]); + expect(built.dependencies.prompts[0].input).toEqual({ + prior: { + type: "built_prompt", + root: { + slug: "summarize-ticket", + }, + }, + }); + }); + + test("validates input and output with the bundled schema DSL", () => { + const typedPrompt = prompt.define({ + slug: "typed", + input: s.object({ + count: s.number(), + }), + output: s.object({ + ok: s.boolean(), + }), + render: ({ input }) => [prompt.user`Count: ${input.count}`], + }); + + expect(() => typedPrompt.build({ count: "nope" })).toThrow( + "input.count must be a number", + ); + expect(() => + typedPrompt.build({ count: 1 }).parseOutput({ ok: "yes" }), + ).toThrow("output.ok must be a boolean"); + }); + + test("translates to an AI SDK shaped object", () => { + const classify = prompt.define({ + slug: "classify", + model: "gpt-4o-mini", + input: s.object({ + text: s.string(), + }), + output: s.object({ + label: s.enum(["bug", "question"]), + }), + render: ({ input }) => [prompt.user`Classify: ${input.text}`], + }); + + expect( + classify + .build({ text: "It crashes" }) + .to(prompt.adapters.aiSDKGenerateObject), + ).toMatchObject({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Classify: It crashes" }], + schema: { + type: "object", + required: ["label"], + }, + experimental_telemetry: { + metadata: { + braintrustPrompt: { + root: { slug: "classify" }, + }, + }, + }, + }); + }); +}); diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts new file mode 100644 index 000000000..8e850b961 --- /dev/null +++ b/js/src/experimental-prompt-api.ts @@ -0,0 +1,816 @@ +type JsonPrimitive = string | number | boolean | null; + +export type PromptJsonSchema = { + type?: string; + properties?: Record; + required?: string[]; + items?: PromptJsonSchema; + enum?: JsonPrimitive[]; + additionalProperties?: boolean; + description?: string; + "x-bt-type"?: string; +}; + +type SchemaParser = (value: unknown, path: string, root: unknown) => T; + +export class PromptSchema { + readonly _type!: TParsed; + readonly _input!: TInput; + + constructor( + private readonly parser: SchemaParser, + private readonly jsonSchema: () => PromptJsonSchema, + public readonly isOptional = false, + ) {} + + parse(value: unknown, path = "value", root: unknown = value): TParsed { + return this.parser(value, path, root); + } + + toJSONSchema(): PromptJsonSchema { + return this.jsonSchema(); + } + + optional(): PromptSchema { + return new PromptSchema( + (value, path, root) => + value === undefined ? undefined : this.parser(value, path, root), + () => this.jsonSchema(), + true, + ); + } +} + +export type InferSchema> = + TSchema extends PromptSchema ? TParsed : never; + +export type InferInputSchema> = + TSchema extends PromptSchema ? TInput : never; + +type SchemaShape = Record>; + +type OptionalParsedKeys = { + [K in keyof TShape]: undefined extends InferSchema ? K : never; +}[keyof TShape]; + +type OptionalInputKeys = { + [K in keyof TShape]: undefined extends InferInputSchema + ? K + : never; +}[keyof TShape]; + +type InferParsedObject = { + [K in keyof TShape as K extends OptionalParsedKeys + ? never + : K]: InferSchema; +} & { + [K in OptionalParsedKeys]?: Exclude< + InferSchema, + undefined + >; +}; + +type InferInputObject = { + [K in keyof TShape as K extends OptionalInputKeys + ? never + : K]: InferInputSchema; +} & { + [K in OptionalInputKeys]?: Exclude< + InferInputSchema, + undefined + >; +}; + +const builtPromptMarker = Symbol("braintrust.experimental_prompt.built"); +const promptDefinitionMarker = Symbol( + "braintrust.experimental_prompt.definition", +); + +export type PromptRole = "system" | "user" | "assistant"; + +export type PromptMessage = { + role: PromptRole; + content: string; +}; + +export type PromptDependencyEntry = { + id?: string; + slug: string; + name?: string; + version?: string; + role: "root" | "include"; + parent?: string; + input: unknown; + metadata?: Record; +}; + +export type PromptDependencies = { + root: { + id?: string; + slug: string; + name?: string; + version?: string; + }; + prompts: PromptDependencyEntry[]; + metadata?: Record; +}; + +export type PromptRenderable = + | PromptMessage + | BuiltPrompt + | readonly PromptRenderable[]; + +export type PromptRenderContext = { + input: TInput; + include: ( + definition: TDefinition, + input: InputOf, + ) => BuiltPrompt, OutputOf>; +}; + +export type PromptDefinitionOptions< + TInputSchema extends PromptSchema, + TOutputSchema extends PromptSchema | undefined = undefined, +> = { + id?: string; + slug: string; + name?: string; + version?: string; + model?: string; + input: TInputSchema; + output?: TOutputSchema; + metadata?: Record; + render: ( + context: PromptRenderContext>, + ) => PromptRenderable | readonly PromptRenderable[]; +}; + +export class PromptDefinition< + TInputSchema extends PromptSchema, + TOutputSchema extends PromptSchema | undefined = undefined, +> { + readonly [promptDefinitionMarker] = true; + readonly id?: string; + readonly slug: string; + readonly name?: string; + readonly version?: string; + readonly model?: string; + readonly inputSchema: TInputSchema; + readonly outputSchema?: TOutputSchema; + readonly metadata?: Record; + + private readonly renderer: PromptDefinitionOptions< + TInputSchema, + TOutputSchema + >["render"]; + + constructor(opts: PromptDefinitionOptions) { + this.id = opts.id; + this.slug = opts.slug; + this.name = opts.name; + this.version = opts.version; + this.model = opts.model; + this.inputSchema = opts.input; + this.outputSchema = opts.output; + this.metadata = opts.metadata; + this.renderer = opts.render; + } + + build( + input: InferInputSchema, + opts: { metadata?: Record } = {}, + ): BuiltPrompt, InferOutput> { + const parsedInput = this.inputSchema.parse( + input, + "input", + input, + ) as InferSchema; + const rendered = this.renderer({ + input: parsedInput, + include: (definition, includeInput) => + definition.build(includeInput) as BuiltPrompt< + ParsedInputOf, + OutputOf + >, + }); + const flattened = flattenRenderable(rendered, this.slug); + const root = { + id: this.id, + slug: this.slug, + name: this.name, + version: this.version, + }; + const promptMetadata = + this.metadata || opts.metadata + ? { ...this.metadata, ...opts.metadata } + : undefined; + const dependencies: PromptDependencies = { + root, + prompts: [ + { + ...root, + role: "root", + input: sanitizeDependencyInput(parsedInput), + metadata: promptMetadata, + }, + ...dedupeDependencyEntries([ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...flattened.dependencies, + ]), + ], + metadata: opts.metadata, + }; + + return new BuiltPrompt< + InferSchema, + InferOutput + >({ + definition: { + model: this.model, + outputSchema: this.outputSchema as + | PromptSchema> + | undefined, + }, + input: parsedInput, + messages: flattened.messages, + dependencies, + }); + } +} + +type AnyPromptDefinition = PromptDefinition< + PromptSchema, + PromptSchema | undefined +>; + +type InferOutput = + TOutputSchema extends PromptSchema + ? TParsed + : unknown; + +export type InputOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + PromptSchema | undefined + > + ? InferInputSchema + : never; + +export type ParsedInputOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + PromptSchema | undefined + > + ? InferSchema + : never; + +export type OutputOf = + TDefinition extends PromptDefinition< + PromptSchema, + infer TOutputSchema + > + ? InferOutput + : never; + +export type PromptInputValue = + | BuiltPrompt, OutputOf> + | Partial> + | undefined; + +export type DynamicPromptInputValue = + | BuiltPrompt + | AnyPromptDefinition + | { + prompt: AnyPromptDefinition; + input?: Record; + }; + +export type BuiltPromptOptions = { + definition: { + model?: string; + outputSchema?: PromptSchema; + }; + input: TInput; + messages: PromptMessage[]; + dependencies: PromptDependencies; +}; + +export type PromptAdapter = ( + builtPrompt: BuiltPrompt, +) => TResult; + +export class BuiltPrompt { + readonly [builtPromptMarker] = true; + readonly definition: { model?: string; outputSchema?: PromptSchema }; + readonly input: TInput; + readonly messages: PromptMessage[]; + readonly dependencies: PromptDependencies; + + constructor(opts: BuiltPromptOptions) { + this.definition = opts.definition; + this.input = opts.input; + this.messages = opts.messages; + this.dependencies = opts.dependencies; + } + + get model(): string | undefined { + return this.definition.model; + } + + get outputJSONSchema(): PromptJsonSchema | undefined { + return this.definition.outputSchema?.toJSONSchema(); + } + + get spanInfo() { + return { + metadata: { + prompt: this.dependencies, + }, + }; + } + + parseOutput(value: unknown): TOutput { + if (!this.definition.outputSchema) { + return value as TOutput; + } + return this.definition.outputSchema.parse(value, "output"); + } + + asText(): string { + return this.messages + .map((message) => `${message.role}: ${message.content}`) + .join("\n"); + } + + to(adapter: PromptAdapter): TResult { + return adapter(this); + } +} + +export type OpenAIChatPromptArgs = { + model?: string; + messages: PromptMessage[]; + response_format?: { + type: "json_schema"; + json_schema: { + name: string; + schema: PromptJsonSchema; + strict: true; + }; + }; + span_info: { + metadata: { + prompt: PromptDependencies; + }; + }; +}; + +export type AISDKGenerateObjectPromptArgs = { + model?: string; + messages: PromptMessage[]; + schema?: PromptJsonSchema; + experimental_telemetry: { + metadata: { + braintrustPrompt: PromptDependencies; + }; + }; +}; + +function openAIChatAdapter( + builtPrompt: BuiltPrompt, +): OpenAIChatPromptArgs { + const outputSchema = builtPrompt.outputJSONSchema; + return { + model: builtPrompt.model, + messages: builtPrompt.messages, + ...(outputSchema + ? { + response_format: { + type: "json_schema" as const, + json_schema: { + name: schemaName(builtPrompt.dependencies.root.slug), + schema: outputSchema, + strict: true as const, + }, + }, + } + : {}), + span_info: builtPrompt.spanInfo, + }; +} + +function aiSDKGenerateObjectAdapter( + builtPrompt: BuiltPrompt, +): AISDKGenerateObjectPromptArgs { + return { + model: builtPrompt.model, + messages: builtPrompt.messages, + schema: builtPrompt.outputJSONSchema, + experimental_telemetry: { + metadata: { + braintrustPrompt: builtPrompt.dependencies, + }, + }, + }; +} + +function definePrompt< + TInputSchema extends PromptSchema, + TOutputSchema extends PromptSchema | undefined = undefined, +>( + opts: PromptDefinitionOptions, +): PromptDefinition { + return new PromptDefinition(opts); +} + +function messageTag(role: PromptRole) { + return ( + strings: TemplateStringsArray, + ...values: readonly unknown[] + ): PromptMessage => ({ + role, + content: renderTaggedTemplate(strings, values), + }); +} + +function renderTaggedTemplate( + strings: TemplateStringsArray, + values: readonly unknown[], +): string { + let rendered = strings[0] ?? ""; + for (let i = 0; i < values.length; i++) { + rendered += stringifyTemplateValue(values[i]) + (strings[i + 1] ?? ""); + } + return rendered; +} + +function stringifyTemplateValue(value: unknown): string { + if (value === undefined || value === null) { + return ""; + } + if (isBuiltPrompt(value)) { + return value.asText(); + } + if (isPromptDefinition(value)) { + return `[prompt:${value.slug}${value.version ? `@${value.version}` : ""}]`; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return String(value); + } + return JSON.stringify(value); +} + +function flattenRenderable( + renderable: PromptRenderable | readonly PromptRenderable[], + parent: string, +): { messages: PromptMessage[]; dependencies: PromptDependencyEntry[] } { + const messages: PromptMessage[] = []; + const dependencies: PromptDependencyEntry[] = []; + const stack = Array.isArray(renderable) ? renderable : [renderable]; + + for (const item of stack) { + if (Array.isArray(item)) { + const flattened = flattenRenderable(item, parent); + messages.push(...flattened.messages); + dependencies.push(...flattened.dependencies); + } else if (isBuiltPrompt(item)) { + messages.push(...item.messages); + dependencies.push( + ...item.dependencies.prompts.map((entry) => ({ + ...entry, + role: "include" as const, + parent, + })), + ); + } else { + messages.push(item); + } + } + + return { messages, dependencies }; +} + +function collectBuiltPromptDependencies( + value: unknown, + parent: string, +): PromptDependencyEntry[] { + if (isBuiltPrompt(value)) { + return value.dependencies.prompts.map((entry) => ({ + ...entry, + role: "include" as const, + parent, + })); + } + if (Array.isArray(value)) { + return value.flatMap((item) => + collectBuiltPromptDependencies(item, parent), + ); + } + if (isRecord(value)) { + return Object.values(value).flatMap((item) => + collectBuiltPromptDependencies(item, parent), + ); + } + return []; +} + +function dedupeDependencyEntries( + entries: PromptDependencyEntry[], +): PromptDependencyEntry[] { + const seen = new Set(); + return entries.filter((entry) => { + const key = JSON.stringify(entry); + if (seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +} + +function sanitizeDependencyInput(value: unknown): unknown { + if (isBuiltPrompt(value)) { + return { + type: "built_prompt", + root: value.dependencies.root, + }; + } + if (isPromptDefinition(value)) { + return { + type: "prompt_definition", + slug: value.slug, + version: value.version, + }; + } + if (Array.isArray(value)) { + return value.map((item) => sanitizeDependencyInput(item)); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + sanitizeDependencyInput(item), + ]), + ); + } + return value; +} + +function schemaName(slug: string): string { + const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); + return name.length > 0 ? `${name}_output` : "prompt_output"; +} + +function isBuiltPrompt(value: unknown): value is BuiltPrompt { + return ( + typeof value === "object" && value !== null && builtPromptMarker in value + ); +} + +function isPromptDefinition(value: unknown): value is AnyPromptDefinition { + return ( + typeof value === "object" && + value !== null && + promptDefinitionMarker in value + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function mergePromptInputs(parent: unknown, overrides: unknown): unknown { + const parentInput = isRecord(parent) ? parent : {}; + if (overrides === undefined) { + return parentInput; + } + if (isRecord(overrides)) { + return { ...parentInput, ...overrides }; + } + return overrides; +} + +function buildAnyPrompt( + definition: AnyPromptDefinition, + input: unknown, +): BuiltPrompt { + return definition.build(input as never) as BuiltPrompt; +} + +function stringSchema(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string") { + throw new Error(`${path} must be a string`); + } + return value; + }, + () => ({ type: "string" }), + ); +} + +function numberSchema(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "number") { + throw new Error(`${path} must be a number`); + } + return value; + }, + () => ({ type: "number" }), + ); +} + +function booleanSchema(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "boolean") { + throw new Error(`${path} must be a boolean`); + } + return value; + }, + () => ({ type: "boolean" }), + ); +} + +function enumSchema( + values: TValues, +): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string" || !values.includes(value)) { + throw new Error(`${path} must be one of ${values.join(", ")}`); + } + return value; + }, + () => ({ type: "string", enum: [...values] }), + ); +} + +function arraySchema>( + item: TItemSchema, +): PromptSchema[], InferInputSchema[]> { + return new PromptSchema( + (value, path, root) => { + if (!Array.isArray(value)) { + throw new Error(`${path} must be an array`); + } + return value.map((itemValue, index) => + item.parse(itemValue, `${path}[${index}]`, root), + ) as InferSchema[]; + }, + () => ({ type: "array", items: item.toJSONSchema() }), + ); +} + +function objectSchema( + shape: TShape, +): PromptSchema, InferInputObject> { + return new PromptSchema( + (value, path, root) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + const record = value as Record; + const rootInput = root === undefined ? record : root; + return Object.fromEntries( + Object.entries(shape) + .filter(([key, schema]) => key in record || !schema.isOptional) + .map(([key, schema]) => [ + key, + schema.parse(record[key], `${path}.${key}`, rootInput), + ]), + ) as InferParsedObject; + }, + () => ({ + type: "object", + properties: Object.fromEntries( + Object.entries(shape).map(([key, schema]) => [ + key, + schema.toJSONSchema(), + ]), + ), + required: Object.entries(shape) + .filter(([, schema]) => !schema.isOptional) + .map(([key]) => key), + additionalProperties: false, + }), + ); +} + +function unknownSchema(): PromptSchema { + return new PromptSchema( + (value) => value, + () => ({}), + ); +} + +function builtPromptSchema(): PromptSchema< + BuiltPrompt +> { + return new PromptSchema( + (value, path) => { + if (!isBuiltPrompt(value)) { + throw new Error(`${path} must be a built prompt`); + } + return value as BuiltPrompt; + }, + () => ({ type: "object", "x-bt-type": "built_prompt" }), + ); +} + +function promptDefinitionSchema< + TDefinition extends AnyPromptDefinition = AnyPromptDefinition, +>(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (!isPromptDefinition(value)) { + throw new Error(`${path} must be a prompt definition`); + } + return value as TDefinition; + }, + () => ({ type: "object", "x-bt-type": "prompt_definition" }), + ); +} + +function promptSchema( + definition: TDefinition, +): PromptSchema< + BuiltPrompt, OutputOf>, + PromptInputValue +>; +function promptSchema(): PromptSchema< + BuiltPrompt, + DynamicPromptInputValue +>; +function promptSchema( + definition?: AnyPromptDefinition, +): PromptSchema, unknown> { + return new PromptSchema( + (value, path, root) => { + if (isBuiltPrompt(value)) { + return value; + } + + let promptDefinition = definition; + let promptInput = value; + if (!promptDefinition) { + if (isPromptDefinition(value)) { + promptDefinition = value; + promptInput = undefined; + } else if (isRecord(value) && isPromptDefinition(value.prompt)) { + promptDefinition = value.prompt; + promptInput = value.input; + } else { + throw new Error(`${path} must be a prompt or built prompt`); + } + } else if (isPromptDefinition(value)) { + promptDefinition = value; + promptInput = undefined; + } else if (isRecord(value) && isPromptDefinition(value.prompt)) { + promptDefinition = value.prompt; + promptInput = value.input; + } + + return buildAnyPrompt( + promptDefinition, + mergePromptInputs(root, promptInput), + ); + }, + () => ({ type: "object", "x-bt-type": "prompt" }), + ); +} + +export const s = { + string: stringSchema, + number: numberSchema, + boolean: booleanSchema, + enum: enumSchema, + array: arraySchema, + object: objectSchema, + unknown: unknownSchema, + // Requires an already-built prompt value; useful when the caller owns construction. + builtPrompt: builtPromptSchema, + // Accepts a built prompt, or builds a prompt definition by merging parent input plus overrides. + prompt: promptSchema, + // Accepts an unbuilt prompt definition as data; render code decides whether/when to build it. + promptDefinition: promptDefinitionSchema, +}; + +export const prompt = { + define: definePrompt, + system: messageTag("system"), + user: messageTag("user"), + assistant: messageTag("assistant"), + asText: (builtPrompt: BuiltPrompt) => builtPrompt.asText(), + isBuiltPrompt, + isPromptDefinition, + adapters: { + openAIChat: openAIChatAdapter, + aiSDKGenerateObject: aiSDKGenerateObjectAdapter, + }, +}; diff --git a/js/src/exports.ts b/js/src/exports.ts index 54cd03e2a..e1b72f7c0 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -262,6 +262,8 @@ export { PromptDefinitionWithTools, } from "./prompt-schemas"; +export { prompt, s } from "./experimental-prompt-api"; + export type { Trace, SpanData, GetThreadOptions } from "./trace"; export { SpanFetcher, CachedSpanFetcher, LocalTrace } from "./trace"; From dd445d2f1fd22f21cff267441d0fe6126afa3306 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Thu, 2 Jul 2026 16:57:04 +0200 Subject: [PATCH 2/6] idk fkin improve everything i guess --- js/src/experimental-prompt-adapters.ts | 104 +++ js/src/experimental-prompt-api.test.ts | 724 +++++++++++---- js/src/experimental-prompt-api.ts | 1134 +++++++++++++++++------- js/src/exports.ts | 2 +- 4 files changed, 1482 insertions(+), 482 deletions(-) create mode 100644 js/src/experimental-prompt-adapters.ts diff --git a/js/src/experimental-prompt-adapters.ts b/js/src/experimental-prompt-adapters.ts new file mode 100644 index 000000000..2ed91e57e --- /dev/null +++ b/js/src/experimental-prompt-adapters.ts @@ -0,0 +1,104 @@ +import type { + PromptDependencies, + PromptJsonSchema, + PromptMessage, +} from "./experimental-prompt-api"; + +type PromptAdapterInput = { + kind: "messages" | "string"; + model?: string; + inputSchema: { toJSONSchema(): PromptJsonSchema }; + outputSchema?: { toJSONSchema(): PromptJsonSchema }; + input: unknown; + messages: PromptMessage[]; + content?: string; + dependencies: PromptDependencies; +}; + +type OpenAIChatPromptArgs = { + model?: string; + messages: PromptMessage[]; + response_format?: { + type: "json_schema"; + json_schema: { + name: string; + schema: PromptJsonSchema; + strict: true; + }; + }; + span_info: { + metadata: { + prompt: PromptDependencies; + }; + }; +}; + +type AISDKGenerateObjectPromptArgs = { + model?: string; + messages: PromptMessage[]; + schema?: PromptJsonSchema; + experimental_telemetry: { + metadata: { + braintrustPrompt: PromptDependencies; + }; + }; +}; + +type OpenAIChatAdapterOptions = Record; +type AISDKGenerateObjectAdapterOptions = Record; + +function schemaName(slug: string): string { + const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); + return name.length > 0 ? `${name}_output` : "prompt_output"; +} + +function openAIChatAdapter( + options: OpenAIChatAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => OpenAIChatPromptArgs { + void options; + return (builtPrompt) => { + const outputSchema = builtPrompt.outputSchema?.toJSONSchema(); + return { + model: builtPrompt.model, + messages: builtPrompt.messages, + ...(outputSchema + ? { + response_format: { + type: "json_schema" as const, + json_schema: { + name: schemaName(builtPrompt.dependencies.root.slug), + schema: outputSchema, + strict: true as const, + }, + }, + } + : undefined), + span_info: { + metadata: { + prompt: builtPrompt.dependencies, + }, + }, + }; + }; +} + +function aiSDKGenerateObjectAdapter( + options: AISDKGenerateObjectAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => AISDKGenerateObjectPromptArgs { + void options; + return (builtPrompt) => ({ + model: builtPrompt.model, + messages: builtPrompt.messages, + schema: builtPrompt.outputSchema?.toJSONSchema(), + experimental_telemetry: { + metadata: { + braintrustPrompt: builtPrompt.dependencies, + }, + }, + }); +} + +export const adapters = { + openAIChat: openAIChatAdapter, + aiSDKGenerateObject: aiSDKGenerateObjectAdapter, +}; diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts index e2d5f0987..0887a63b2 100644 --- a/js/src/experimental-prompt-api.test.ts +++ b/js/src/experimental-prompt-api.test.ts @@ -1,43 +1,43 @@ import { describe, expect, test } from "vitest"; -import { - prompt, - s, - type InputOf, - type OutputOf, -} from "./experimental-prompt-api"; +import { prompt } from "./experimental-prompt-api"; describe("experimental prompt API", () => { - test("builds typed prompts and translates to OpenAI chat args", () => { + test("builds message prompts and translates to OpenAI chat args", () => { const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", - input: s.object({ - ticket: s.string(), - }), - output: s.object({ - subject: s.string(), - body: s.string(), - urgency: s.enum(["low", "medium", "high"]), - }), + input: (s) => + s.object({ + ticket: s.string(), + }), + output: (s) => + s.object({ + subject: s.string(), + body: s.string(), + urgency: s.enum(["low", "medium", "high"]), + }), render: ({ input }) => [ prompt.system`You write concise support replies.`, prompt.user`Ticket: ${input.ticket}`, ], }); - type SupportReplyInput = InputOf; - type SupportReplyOutput = OutputOf; - - const input: SupportReplyInput = { ticket: "I cannot find eval history." }; - const output: SupportReplyOutput = { + const built = supportReply.build({ + ticket: "I cannot find eval history.", + }); + const output = { subject: "Finding eval history", body: "Here is where to look.", urgency: "low", }; - const built = supportReply.build(input); - expect(built.parseOutput(output)).toEqual(output); - expect(built.to(prompt.adapters.openAIChat)).toMatchObject({ + expect(built.kind).toBe("messages"); + expect("content" in built).toBe(false); + expect([...built]).toEqual(built.messages); + expect(built.definition.outputSchema?.parse(output, "output")).toEqual( + output, + ); + expect(built.to(prompt.adapters.openAIChat())).toMatchObject({ model: "gpt-4o", messages: [ { role: "system", content: "You write concise support replies." }, @@ -62,79 +62,170 @@ describe("experimental prompt API", () => { { slug: "support-reply", role: "root", - input, + input: { ticket: "I cannot find eval history." }, }, ], }, }, }, }); + + if (false) { + // @ts-expect-error message prompts do not expose string content + void built.content; + } }); - test("includes nested prompts and records prompt dependencies", () => { - const brandVoice = prompt.define({ - slug: "brand-voice", - version: "v3", - input: s.object({ - company: s.string(), - }), - render: ({ input }) => [ - prompt.system`Use ${input.company}'s voice. Be warm and direct.`, - ], + test("builds string prompts and coerces adapters to a user message", () => { + const policyText = prompt.define({ + slug: "policy-text", + model: "gpt-4o-mini", + input: (s) => + s.object({ + policy: s.string(), + }), + render: ({ input }) => prompt.text`Policy: ${input.policy}`, }); - const supportReply = prompt.define({ - slug: "support-reply", - version: "v8", - input: s.object({ - company: s.string(), - ticket: s.string(), - }), - render: ({ input, include }) => [ - include(brandVoice, { company: input.company }), - prompt.user`Draft a reply for: ${input.ticket}`, - ], + const built = policyText.build({ policy: "Prefer short answers." }); + + expect(built.kind).toBe("string"); + expect(built.content).toBe("Policy: Prefer short answers."); + expect("messages" in built).toBe(false); + expect(built.to(prompt.adapters.openAIChat())).toMatchObject({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Policy: Prefer short answers." }], + span_info: { + metadata: { + prompt: { + root: { slug: "policy-text" }, + }, + }, + }, + }); + expect( + built.to((snapshot) => ({ + kind: snapshot.kind, + content: snapshot.kind === "string" ? snapshot.content : undefined, + messages: snapshot.messages, + })), + ).toEqual({ + kind: "string", + content: "Policy: Prefer short answers.", + messages: [{ role: "user", content: "Policy: Prefer short answers." }], }); - const built = supportReply.build({ - company: "Braintrust", - ticket: "Where did my experiment go?", + if (false) { + // @ts-expect-error string prompts do not expose messages + void built.messages; + // @ts-expect-error string prompts are not iterable + void [...built]; + } + }); + + test("extends adapter args with a typed deep merge", () => { + const classify = prompt.define({ + slug: "classify", + model: "gpt-4o-mini", + input: (s) => + s.object({ + text: s.string(), + }), + output: (s) => + s.object({ + label: s.enum(["bug", "question"]), + }), + render: ({ input }) => [prompt.user`Classify: ${input.text}`], }); - expect(built.messages).toEqual([ - { - role: "system", - content: "Use Braintrust's voice. Be warm and direct.", + const built = classify.build({ text: "It crashes" }); + const args = built.to(prompt.adapters.openAIChat()); + const extended = args.extend({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, }, - { - role: "user", - content: "Draft a reply for: Where did my experiment go?", + }); + const extendedAgain = extended.extend({ + top_p: 0.5, + span_info: { + metadata: { + requestId: "req_123", + }, }, - ]); - expect(built.dependencies.prompts).toMatchObject([ - { - slug: "support-reply", - version: "v8", - role: "root", + }); + + expect(Object.keys(args)).not.toContain("extend"); + expect({ ...args }).not.toHaveProperty("extend"); + expect(extended).toMatchObject({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + prompt: { + root: { slug: "classify" }, + }, + }, }, - { - slug: "brand-voice", - version: "v3", - role: "include", - parent: "support-reply", - input: { company: "Braintrust" }, + }); + expect(extendedAgain).toMatchObject({ + temperature: 0.2, + top_p: 0.5, + span_info: { + metadata: { + caller: "support-workflow", + requestId: "req_123", + prompt: { + root: { slug: "classify" }, + }, + }, }, - ]); + }); + expect(() => built.to(() => "nope" as never)).toThrow( + "prompt adapters must return an object", + ); + expect(() => built.to(() => [] as never)).toThrow( + "prompt adapters must return an object", + ); + expect(() => args.extend("nope" as never)).toThrow( + "extend must receive an object", + ); + + if (false) { + // @ts-expect-error adapters must return objects + void built.to(() => "nope"); + + const typedArgs = built.to(prompt.adapters.openAIChat()).extend({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, + }, + }); + const temperature: number = typedArgs.temperature; + const caller: string = typedArgs.span_info.metadata.caller; + const slug: string = typedArgs.span_info.metadata.prompt.root.slug; + void temperature; + void caller; + void slug; + + // @ts-expect-error extend only accepts objects + void typedArgs.extend("nope"); + } }); - test("builds schema prompt fields from merged parent input and overrides", () => { + test("auto-builds message prompt inputs and preserves spread dependencies", () => { const brandVoice = prompt.define({ slug: "brand-voice", version: "v3", - input: s.object({ - company: s.string(), - tone: s.string(), - }), + input: (s) => + s.object({ + company: s.string(), + tone: s.string(), + }), render: ({ input }) => [ prompt.system`Use ${input.company}'s ${input.tone} voice.`, ], @@ -143,23 +234,51 @@ describe("experimental prompt API", () => { const supportReply = prompt.define({ slug: "support-reply", version: "v8", - input: s.object({ - company: s.string(), - ticket: s.string(), - voice: s.prompt(brandVoice), - }), + input: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + voice: s.messagesPromptDefinition(brandVoice), + }), render: ({ input }) => [ - input.voice, + ...input.voice, prompt.user`Draft a reply for: ${input.ticket}`, ], }); - type SupportReplyInput = InputOf; + type SupportReplyInput = Parameters[0]; const input: SupportReplyInput = { company: "Braintrust", ticket: "Where did my experiment go?", voice: { tone: "direct" }, }; + const overriddenNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { company: "Acme", tone: "direct" }, + }; + void overriddenNestedPromptInput; + const missingNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error nested prompt input still needs fields that are not supplied by the parent input + voice: {}, + }; + void missingNestedPromptInput; + const unbuiltNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error raw prompt definitions are not prompt inputs + voice: brandVoice, + }; + void unbuiltNestedPromptInput; + const dynamicPromptDefinitionInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error prompt definition payloads are not prompt inputs + voice: { prompt: brandVoice, input: { tone: "direct" } }, + }; + void dynamicPromptDefinitionInput; const built = supportReply.build(input); @@ -181,7 +300,7 @@ describe("experimental prompt API", () => { company: "Braintrust", ticket: "Where did my experiment go?", voice: { - type: "built_prompt", + type: "built_messages_prompt", root: { slug: "brand-voice", version: "v3" }, }, }, @@ -197,143 +316,428 @@ describe("experimental prompt API", () => { }, }, ]); + expect(() => + supportReply.build({ + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: brandVoice as never, + }), + ).toThrow("input.voice must be a built messages prompt or prompt input"); }); - test("allows dynamic s.prompt fields that receive a prompt definition", () => { - const brandVoice = prompt.define({ - slug: "brand-voice", - input: s.object({ - company: s.string(), - tone: s.string(), - }), - render: ({ input }) => [ - prompt.system`Use ${input.company}'s ${input.tone} voice.`, - ], + test("auto-builds string prompt inputs and preserves interpolation dependencies", () => { + const policyText = prompt.define({ + slug: "policy-text", + version: "v2", + input: (s) => + s.object({ + company: s.string(), + policy: s.string(), + }), + render: ({ input }) => prompt.text`${input.company}: ${input.policy}`, }); const supportReply = prompt.define({ slug: "support-reply", - input: s.object({ - company: s.string(), - ticket: s.string(), - voice: s.prompt(), - }), + input: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + policy: s.stringPromptDefinition(policyText), + }), render: ({ input }) => [ - input.voice, + prompt.system`Follow this policy: ${input.policy}`, prompt.user`Draft a reply for: ${input.ticket}`, ], }); - const built = supportReply.build({ + type SupportReplyInput = Parameters[0]; + const input: SupportReplyInput = { company: "Braintrust", ticket: "Where did my experiment go?", - voice: { - prompt: brandVoice, - input: { tone: "calm" }, - }, - }); + policy: { policy: "Be concise." }, + }; + const invalidPolicyInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error nested string prompt input still needs fields that are not supplied by the parent input + policy: {}, + }; + void invalidPolicyInput; + + const built = supportReply.build(input); expect(built.messages).toEqual([ { role: "system", - content: "Use Braintrust's calm voice.", + content: "Follow this policy: Braintrust: Be concise.", }, { role: "user", content: "Draft a reply for: Where did my experiment go?", }, ]); + expect(built.dependencies.prompts).toMatchObject([ + { + slug: "support-reply", + role: "root", + input: { + company: "Braintrust", + ticket: "Where did my experiment go?", + policy: { + type: "built_string_prompt", + root: { slug: "policy-text", version: "v2" }, + }, + }, + }, + { + slug: "policy-text", + version: "v2", + role: "include", + parent: "support-reply", + input: { + company: "Braintrust", + policy: "Be concise.", + }, + }, + ]); }); - test("accepts a built prompt as input and sanitizes dependencies", () => { - const summarizeTicket = prompt.define({ - slug: "summarize-ticket", - input: s.object({ - ticket: s.string(), - }), - render: ({ input }) => [prompt.user`Summarize: ${input.ticket}`], + test("requires matching built prompt kinds for dynamic schemas", () => { + const messagePrompt = prompt.define({ + slug: "message-prompt", + input: (s) => + s.object({ + topic: s.string(), + }), + render: ({ input }) => [prompt.user`Message about ${input.topic}`], + }); + const stringPrompt = prompt.define({ + slug: "string-prompt", + input: (s) => + s.object({ + topic: s.string(), + }), + render: ({ input }) => prompt.text`String about ${input.topic}`, }); - const critiquePrompt = prompt.define({ - slug: "critique-prompt", - input: s.object({ - prior: s.builtPrompt(), - }), + const consumeBoth = prompt.define({ + slug: "consume-both", + input: (s) => + s.object({ + messagePart: s.builtMessagesPrompt(), + stringPart: s.builtStringPrompt(), + }), render: ({ input }) => [ - prompt.user`Critique this prompt:\n${prompt.asText(input.prior)}`, + ...input.messagePart, + prompt.user`Fragment: ${input.stringPart}`, ], }); - const prior = summarizeTicket.build({ ticket: "Billing looks wrong." }); - const built = critiquePrompt.build({ prior }); + const messagePart = messagePrompt.build({ topic: "tracing" }); + const stringPart = stringPrompt.build({ topic: "evals" }); - expect(built.messages).toEqual([ - { - role: "user", - content: "Critique this prompt:\nuser: Summarize: Billing looks wrong.", + type ConsumeBothInput = Parameters[0]; + const wrongMessageKind: ConsumeBothInput = { + // @ts-expect-error string prompts cannot satisfy message prompt schemas + messagePart: stringPart, + stringPart, + }; + void wrongMessageKind; + const wrongStringKind: ConsumeBothInput = { + messagePart, + // @ts-expect-error message prompts cannot satisfy string prompt schemas + stringPart: messagePart, + }; + void wrongStringKind; + const rawDefinitionInput: ConsumeBothInput = { + // @ts-expect-error raw prompt definitions are not prompt inputs + messagePart: messagePrompt, + stringPart, + }; + void rawDefinitionInput; + const dynamicPayloadInput: ConsumeBothInput = { + messagePart: { + // @ts-expect-error prompt definition payloads are not prompt inputs + prompt: messagePrompt, + input: { topic: "tracing" }, }, + stringPart, + }; + void dynamicPayloadInput; + + const built = consumeBoth.build({ messagePart, stringPart }); + + expect(built.messages).toEqual([ + { role: "user", content: "Message about tracing" }, + { role: "user", content: "Fragment: String about evals" }, ]); - expect(built.dependencies.prompts[0].input).toEqual({ - prior: { - type: "built_prompt", - root: { - slug: "summarize-ticket", - }, - }, + expect(() => + consumeBoth.build({ messagePart: stringPart as never, stringPart }), + ).toThrow("input.messagePart must be a built messages prompt"); + expect(() => + consumeBoth.build({ messagePart, stringPart: messagePart as never }), + ).toThrow("input.stringPart must be a built string prompt"); + }); + + test("preserves dependencies through spread and interpolation outside schema inputs", () => { + const messagePrompt = prompt.define({ + slug: "message-prompt", + input: (s) => + s.object({ + topic: s.string(), + }), + render: ({ input }) => [prompt.user`Message about ${input.topic}`], + }); + const stringPrompt = prompt.define({ + slug: "string-prompt", + input: (s) => + s.object({ + topic: s.string(), + }), + render: ({ input }) => prompt.text`String about ${input.topic}`, }); + + const messagePart = messagePrompt.build({ topic: "tracing" }); + const stringPart = stringPrompt.build({ topic: "evals" }); + const wrapper = prompt.define({ + slug: "wrapper", + input: (s) => s.object({}), + render: () => [...messagePart, prompt.user`Fragment: ${stringPart}`], + }); + + const built = wrapper.build({}); + + expect(built.dependencies.prompts).toMatchObject([ + { slug: "wrapper", role: "root" }, + { slug: "message-prompt", role: "include", parent: "wrapper" }, + { slug: "string-prompt", role: "include", parent: "wrapper" }, + ]); }); - test("validates input and output with the bundled schema DSL", () => { + test("validates input, output, and render shapes", () => { const typedPrompt = prompt.define({ slug: "typed", - input: s.object({ - count: s.number(), - }), - output: s.object({ - ok: s.boolean(), - }), + input: (s) => + s.object({ + count: s.number(), + }), + output: (s) => + s.object({ + ok: s.boolean(), + }), render: ({ input }) => [prompt.user`Count: ${input.count}`], }); + const invalidRenderPrompt = prompt.define({ + slug: "invalid-render", + input: (s) => s.object({}), + // @ts-expect-error render must return a message array or prompt.text + render: () => prompt.user`Nope`, + }); - expect(() => typedPrompt.build({ count: "nope" })).toThrow( - "input.count must be a number", - ); expect(() => - typedPrompt.build({ count: 1 }).parseOutput({ ok: "yes" }), + // @ts-expect-error runtime validation rejects invalid input too + typedPrompt.build({ count: "nope" }), + ).toThrow("input.count must be a number"); + const built = typedPrompt.build({ count: 1 }); + expect(() => + built.definition.outputSchema?.parse({ ok: "yes" }, "output"), ).toThrow("output.ok must be a boolean"); + expect(() => invalidRenderPrompt.build({})).toThrow( + "render must return a message array or prompt.text", + ); }); - test("translates to an AI SDK shaped object", () => { + test("passes scoped schema helpers to input and output callbacks", () => { + let inputHelperKeys: string[] = []; + let outputHelperKeys: string[] = []; + const typedPrompt = prompt.define({ + slug: "schema-helper-scopes", + input: (s) => { + inputHelperKeys = Object.keys(s).sort(); + return s.object({ + topic: s.string(), + }); + }, + output: (s) => { + outputHelperKeys = Object.keys(s).sort(); + return s.object({ + ok: s.boolean(), + }); + }, + render: ({ input }) => [prompt.user`Topic: ${input.topic}`], + }); + + expect(inputHelperKeys).toContain("builtMessagesPrompt"); + expect(inputHelperKeys).toContain("stringPromptDefinition"); + expect(outputHelperKeys).toContain("object"); + expect(outputHelperKeys).not.toContain("builtMessagesPrompt"); + expect(outputHelperKeys).not.toContain("builtStringPrompt"); + expect(outputHelperKeys).not.toContain("messagesPromptDefinition"); + expect(outputHelperKeys).not.toContain("stringPromptDefinition"); + + if (false) { + prompt.define({ + slug: "raw-input-schema", + // @ts-expect-error input must be a schema function + input: typedPrompt.inputSchema, + render: () => [prompt.user`Nope`], + }); + + prompt.define({ + slug: "raw-output-schema", + input: (s) => s.object({}), + // @ts-expect-error output must be a schema function + output: typedPrompt.outputSchema, + render: () => [prompt.user`Nope`], + }); + + prompt.define({ + slug: "prompt-output-field", + input: (s) => s.object({}), + output: (s) => + s.object({ + // @ts-expect-error output schema helpers do not include prompt helpers + prompt: s.builtMessagesPrompt(), + }), + render: () => [prompt.user`Nope`], + }); + + prompt.define({ + slug: "prompt-output-array", + input: (s) => s.object({}), + output: (s) => + // @ts-expect-error output schema helpers do not include prompt helpers + s.array(s.builtStringPrompt()), + render: () => [prompt.user`Nope`], + }); + } + + expect(() => + prompt.define({ + slug: "runtime-raw-input-schema", + input: typedPrompt.inputSchema as never, + render: () => [prompt.user`Nope`], + }), + ).toThrow("input must be a schema function"); + expect(() => + prompt.define({ + slug: "runtime-raw-output-schema", + input: (s) => s.object({}), + output: typedPrompt.outputSchema as never, + render: () => [prompt.user`Nope`], + }), + ).toThrow("output must be a schema function"); + }); + + test("passes flat prompt snapshots to custom adapters", () => { const classify = prompt.define({ slug: "classify", model: "gpt-4o-mini", - input: s.object({ - text: s.string(), - }), - output: s.object({ - label: s.enum(["bug", "question"]), - }), + input: (s) => + s.object({ + text: s.string(), + }), + output: (s) => + s.object({ + label: s.enum(["bug", "question"]), + }), render: ({ input }) => [prompt.user`Classify: ${input.text}`], }); + const summarize = prompt.define({ + slug: "summarize", + input: (s) => + s.object({ + text: s.string(), + }), + render: ({ input }) => prompt.text`Summarize: ${input.text}`, + }); expect( - classify - .build({ text: "It crashes" }) - .to(prompt.adapters.aiSDKGenerateObject), + classify.build({ text: "It crashes" }).to((snapshot) => ({ + keys: Object.keys(snapshot).sort(), + kind: snapshot.kind, + inputSchema: snapshot.inputSchema.toJSONSchema(), + outputSchema: snapshot.outputSchema?.toJSONSchema(), + input: snapshot.input, + })), ).toMatchObject({ - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Classify: It crashes" }], - schema: { + keys: [ + "dependencies", + "input", + "inputSchema", + "kind", + "messages", + "model", + "outputSchema", + ], + kind: "messages", + inputSchema: { + type: "object", + required: ["text"], + }, + outputSchema: { type: "object", required: ["label"], }, - experimental_telemetry: { - metadata: { - braintrustPrompt: { - root: { slug: "classify" }, - }, - }, + input: { text: "It crashes" }, + }); + expect( + summarize.build({ text: "It crashes" }).to((snapshot) => ({ + keys: Object.keys(snapshot).sort(), + kind: snapshot.kind, + content: snapshot.kind === "string" ? snapshot.content : undefined, + messages: snapshot.messages, + })), + ).toEqual({ + keys: [ + "content", + "dependencies", + "input", + "inputSchema", + "kind", + "messages", + "model", + "outputSchema", + ], + kind: "string", + content: "Summarize: It crashes", + messages: [{ role: "user", content: "Summarize: It crashes" }], + }); + }); + + test("input schema helper exposes only the explicit built prompt helpers", () => { + prompt.define({ + slug: "input-helper-surface", + input: (s) => { + expect("builtMessagesPrompt" in s).toBe(true); + expect("builtStringPrompt" in s).toBe(true); + expect("messagesPromptDefinition" in s).toBe(true); + expect("stringPromptDefinition" in s).toBe(true); + expect("prompt" in s).toBe(false); + expect("builtPrompt" in s).toBe(false); + expect("promptDefinition" in s).toBe(false); + + if (false) { + // @ts-expect-error built message prompt schemas only accept already-built prompts + void s.builtMessagesPrompt(undefined); + // @ts-expect-error built string prompt schemas only accept already-built prompts + void s.builtStringPrompt(undefined); + // @ts-expect-error message prompt definition schemas require a definition + void s.messagesPromptDefinition(); + // @ts-expect-error string prompt definition schemas require a definition + void s.stringPromptDefinition(); + // @ts-expect-error old generic prompt schema helper was removed + void s.prompt; + // @ts-expect-error old generic built prompt schema helper was removed + void s.builtPrompt; + // @ts-expect-error prompt definitions are no longer accepted as schema inputs + void s.promptDefinition; + } + + return s.object({}); }, + render: () => [prompt.user`Ok`], }); }); }); diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts index 8e850b961..c2e23baf2 100644 --- a/js/src/experimental-prompt-api.ts +++ b/js/src/experimental-prompt-api.ts @@ -1,3 +1,5 @@ +import { adapters } from "./experimental-prompt-adapters"; + type JsonPrimitive = string | number | boolean | null; export type PromptJsonSchema = { @@ -13,9 +15,18 @@ export type PromptJsonSchema = { type SchemaParser = (value: unknown, path: string, root: unknown) => T; -export class PromptSchema { +type SchemaDomain = "input" | "output"; + +class PromptSchema< + TParsed, + TInput = TParsed, + TKind = unknown, + TDomain extends SchemaDomain = "input", +> { readonly _type!: TParsed; readonly _input!: TInput; + readonly _kind!: TKind; + readonly _domain!: TDomain; constructor( private readonly parser: SchemaParser, @@ -31,8 +42,18 @@ export class PromptSchema { return this.jsonSchema(); } - optional(): PromptSchema { - return new PromptSchema( + optional(): PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + > { + return new PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + >( (value, path, root) => value === undefined ? undefined : this.parser(value, path, root), () => this.jsonSchema(), @@ -41,20 +62,34 @@ export class PromptSchema { } } -export type InferSchema> = - TSchema extends PromptSchema ? TParsed : never; +type AnySchema = PromptSchema; +type InputSchema = PromptSchema; +type OutputSchema = PromptSchema; + +type InferSchema = + TSchema extends PromptSchema + ? TParsed + : never; -export type InferInputSchema> = - TSchema extends PromptSchema ? TInput : never; +type InferInputSchema = + TSchema extends PromptSchema + ? TInput + : never; -type SchemaShape = Record>; +type SchemaShape = Record; +type InputSchemaShape = Record; +type OutputSchemaShape = Record; type OptionalParsedKeys = { [K in keyof TShape]: undefined extends InferSchema ? K : never; }[keyof TShape]; type OptionalInputKeys = { - [K in keyof TShape]: undefined extends InferInputSchema + [K in keyof TShape]: undefined extends InferObjectInputSchema< + TShape[K], + TShape, + K + > ? K : never; }[keyof TShape]; @@ -73,27 +108,59 @@ type InferParsedObject = { type InferInputObject = { [K in keyof TShape as K extends OptionalInputKeys ? never - : K]: InferInputSchema; + : K]: InferObjectInputSchema; } & { [K in OptionalInputKeys]?: Exclude< - InferInputSchema, + InferObjectInputSchema, undefined >; }; +type InferObjectInputSchema< + TSchema extends AnySchema, + TShape extends SchemaShape, + TKey extends keyof TShape, +> = + TSchema extends PromptSchema + ? TKind extends PromptFieldKind + ? PromptInputValueForObject< + TDefinition, + TPromptKind, + TShape, + TKey, + TInput + > + : TInput + : never; + const builtPromptMarker = Symbol("braintrust.experimental_prompt.built"); const promptDefinitionMarker = Symbol( "braintrust.experimental_prompt.definition", ); +const promptTextMarker = Symbol("braintrust.experimental_prompt.text"); +const promptDependencyMarker = Symbol( + "braintrust.experimental_prompt.dependencies", +); -export type PromptRole = "system" | "user" | "assistant"; +type PromptRole = "system" | "user" | "assistant"; +type PromptKind = "messages" | "string"; export type PromptMessage = { role: PromptRole; content: string; }; -export type PromptDependencyEntry = { +type PromptMessageWithDependencies = PromptMessage & { + readonly [promptDependencyMarker]?: PromptDependencyEntry[]; +}; + +type PromptText = { + readonly [promptTextMarker]: true; + readonly content: string; + readonly dependencies: PromptDependencyEntry[]; +}; + +type PromptDependencyEntry = { id?: string; slug: string; name?: string; @@ -101,7 +168,6 @@ export type PromptDependencyEntry = { role: "root" | "include"; parent?: string; input: unknown; - metadata?: Record; }; export type PromptDependencies = { @@ -112,42 +178,42 @@ export type PromptDependencies = { version?: string; }; prompts: PromptDependencyEntry[]; - metadata?: Record; }; -export type PromptRenderable = - | PromptMessage - | BuiltPrompt - | readonly PromptRenderable[]; +type PromptRenderResult = readonly PromptMessage[] | PromptText; -export type PromptRenderContext = { +type PromptRenderContext = { input: TInput; include: ( definition: TDefinition, input: InputOf, - ) => BuiltPrompt, OutputOf>; + ) => BuiltPromptOf; }; -export type PromptDefinitionOptions< - TInputSchema extends PromptSchema, - TOutputSchema extends PromptSchema | undefined = undefined, +type InputSchemaHelpers = typeof inputSchemaHelpers; +type OutputSchemaHelpers = typeof outputSchemaHelpers; + +type PromptDefinitionOptions< + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined, + TRenderResult extends PromptRenderResult, > = { id?: string; slug: string; name?: string; version?: string; model?: string; - input: TInputSchema; - output?: TOutputSchema; - metadata?: Record; + input: (s: InputSchemaHelpers) => TInputSchema; + output?: (s: OutputSchemaHelpers) => TOutputSchema; render: ( context: PromptRenderContext>, - ) => PromptRenderable | readonly PromptRenderable[]; + ) => TRenderResult; }; -export class PromptDefinition< - TInputSchema extends PromptSchema, - TOutputSchema extends PromptSchema | undefined = undefined, +class PromptDefinition< + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined, + TRenderResult extends PromptRenderResult, > { readonly [promptDefinitionMarker] = true; readonly id?: string; @@ -157,29 +223,39 @@ export class PromptDefinition< readonly model?: string; readonly inputSchema: TInputSchema; readonly outputSchema?: TOutputSchema; - readonly metadata?: Record; private readonly renderer: PromptDefinitionOptions< TInputSchema, - TOutputSchema + TOutputSchema, + TRenderResult >["render"]; - constructor(opts: PromptDefinitionOptions) { + constructor( + opts: PromptDefinitionOptions, + ) { this.id = opts.id; this.slug = opts.slug; this.name = opts.name; this.version = opts.version; this.model = opts.model; - this.inputSchema = opts.input; - this.outputSchema = opts.output; - this.metadata = opts.metadata; + if (typeof opts.input !== "function") { + throw new Error("input must be a schema function"); + } + if (opts.output !== undefined && typeof opts.output !== "function") { + throw new Error("output must be a schema function"); + } + this.inputSchema = opts.input(inputSchemaHelpers); + this.outputSchema = opts.output?.(outputSchemaHelpers); this.renderer = opts.render; } build( input: InferInputSchema, - opts: { metadata?: Record } = {}, - ): BuiltPrompt, InferOutput> { + ): BuiltPromptForRenderResult< + InferSchema, + InferOutput, + TRenderResult + > { const parsedInput = this.inputSchema.parse( input, "input", @@ -188,238 +264,424 @@ export class PromptDefinition< const rendered = this.renderer({ input: parsedInput, include: (definition, includeInput) => - definition.build(includeInput) as BuiltPrompt< - ParsedInputOf, - OutputOf - >, + definition.build(includeInput) as BuiltPromptOf, }); - const flattened = flattenRenderable(rendered, this.slug); const root = { id: this.id, slug: this.slug, name: this.name, version: this.version, }; - const promptMetadata = - this.metadata || opts.metadata - ? { ...this.metadata, ...opts.metadata } - : undefined; - const dependencies: PromptDependencies = { - root, - prompts: [ - { - ...root, - role: "root", - input: sanitizeDependencyInput(parsedInput), - metadata: promptMetadata, + + if (isPromptText(rendered)) { + const dependencies = createPromptDependencies(root, parsedInput, [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectDependencyEntries(rendered.dependencies, this.slug), + ]); + + return new BuiltStringPrompt< + InferSchema, + InferOutput + >({ + definition: { + model: this.model, + inputSchema: this.inputSchema as PromptSchema< + InferSchema, + unknown, + unknown, + "input" + >, + outputSchema: this.outputSchema as + | PromptSchema< + InferOutput, + unknown, + unknown, + "output" + > + | undefined, }, - ...dedupeDependencyEntries([ - ...collectBuiltPromptDependencies(parsedInput, this.slug), - ...flattened.dependencies, - ]), - ], - metadata: opts.metadata, - }; + input: parsedInput, + content: rendered.content, + dependencies, + }) as BuiltPromptForRenderResult< + InferSchema, + InferOutput, + TRenderResult + >; + } + + if (!Array.isArray(rendered)) { + throw new Error("render must return a message array or prompt.text"); + } + + const messages = rendered.map((message, index) => + assertPromptMessage(message, `render[${index}]`), + ); + const dependencies = createPromptDependencies(root, parsedInput, [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectMessageDependencies(messages, this.slug), + ]); - return new BuiltPrompt< + return new BuiltMessagesPrompt< InferSchema, InferOutput >({ definition: { model: this.model, + inputSchema: this.inputSchema as PromptSchema< + InferSchema, + unknown, + unknown, + "input" + >, outputSchema: this.outputSchema as - | PromptSchema> + | PromptSchema, unknown, unknown, "output"> | undefined, }, input: parsedInput, - messages: flattened.messages, + messages, dependencies, - }); + }) as BuiltPromptForRenderResult< + InferSchema, + InferOutput, + TRenderResult + >; } } type AnyPromptDefinition = PromptDefinition< - PromptSchema, - PromptSchema | undefined + InputSchema, + OutputSchema | undefined, + PromptRenderResult +>; + +type AnyMessagesPromptDefinition = PromptDefinition< + InputSchema, + OutputSchema | undefined, + readonly PromptMessage[] +>; + +type AnyStringPromptDefinition = PromptDefinition< + InputSchema, + OutputSchema | undefined, + PromptText >; type InferOutput = - TOutputSchema extends PromptSchema + TOutputSchema extends PromptSchema ? TParsed : unknown; -export type InputOf = +type InputOf = TDefinition extends PromptDefinition< infer TInputSchema, - PromptSchema | undefined + OutputSchema | undefined, + PromptRenderResult > ? InferInputSchema : never; -export type ParsedInputOf = +type ParsedInputOf = TDefinition extends PromptDefinition< infer TInputSchema, - PromptSchema | undefined + OutputSchema | undefined, + PromptRenderResult > ? InferSchema : never; -export type OutputOf = +type OutputOf = TDefinition extends PromptDefinition< - PromptSchema, - infer TOutputSchema + InputSchema, + infer TOutputSchema, + PromptRenderResult > ? InferOutput : never; -export type PromptInputValue = - | BuiltPrompt, OutputOf> - | Partial> - | undefined; +type BuiltPromptOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + infer TOutputSchema, + infer TRenderResult + > + ? BuiltPromptForRenderResult< + InferSchema, + InferOutput, + TRenderResult + > + : never; -export type DynamicPromptInputValue = - | BuiltPrompt - | AnyPromptDefinition - | { - prompt: AnyPromptDefinition; - input?: Record; - }; +type BuiltPromptForRenderResult = + TRenderResult extends PromptText + ? BuiltStringPrompt + : TRenderResult extends readonly PromptMessage[] + ? BuiltMessagesPrompt + : never; + +type BuiltPromptForKind< + TInput, + TOutput, + TKind extends PromptKind, +> = TKind extends "messages" + ? BuiltMessagesPrompt + : BuiltStringPrompt; + +type PromptFieldKind< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, +> = { + type: "prompt"; + definition: TDefinition; + promptKind: TPromptKind; +}; -export type BuiltPromptOptions = { +type PromptInputOverrides< + TInput, + TParentKeys extends PropertyKey, +> = TInput extends object + ? Omit & + Partial>> + : TInput; + +type PromptInputValue< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, +> = + | BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TPromptKind + > + | InputOf; + +type PromptInputValueForObject< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, + TShape extends SchemaShape, + TKey extends keyof TShape, + TInput, +> = + | BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TPromptKind + > + | PromptInputOverrides, Exclude> + | (undefined extends TInput ? undefined : never); + +type BuiltPromptOptions = { definition: { model?: string; - outputSchema?: PromptSchema; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; }; input: TInput; + dependencies: PromptDependencies; +}; + +type BuiltMessagesPromptOptions = BuiltPromptOptions< + TInput, + TOutput +> & { messages: PromptMessage[]; +}; + +type BuiltStringPromptOptions = BuiltPromptOptions< + TInput, + TOutput +> & { + content: string; +}; + +type PromptAdapterInput = { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + input: TInput; dependencies: PromptDependencies; +} & ( + | { + kind: "messages"; + messages: PromptMessage[]; + } + | { + kind: "string"; + content: string; + messages: PromptMessage[]; + } +); + +type Simplify = { [K in keyof T]: T[K] } & {}; + +type DeepMerge = Simplify< + Omit & { + [K in keyof TExtension]: K extends keyof TBase + ? DeepMergeValue + : TExtension[K]; + } +>; + +type DeepMergeValue = TBase extends readonly unknown[] + ? TExtension + : TExtension extends readonly unknown[] + ? TExtension + : TBase extends object + ? TExtension extends object + ? DeepMerge + : TExtension + : TExtension; + +type PromptExtension = Record; +type PromptAdapterResult = PromptExtension; + +type Extendable = T & { + extend( + extension: TExtension, + ): Extendable>; }; -export type PromptAdapter = ( - builtPrompt: BuiltPrompt, +type PromptAdapter = ( + builtPrompt: PromptAdapterInput, ) => TResult; -export class BuiltPrompt { +class BuiltMessagesPrompt implements Iterable { readonly [builtPromptMarker] = true; - readonly definition: { model?: string; outputSchema?: PromptSchema }; + readonly kind = "messages"; + readonly definition: { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + }; readonly input: TInput; readonly messages: PromptMessage[]; readonly dependencies: PromptDependencies; - constructor(opts: BuiltPromptOptions) { + constructor(opts: BuiltMessagesPromptOptions) { this.definition = opts.definition; this.input = opts.input; this.messages = opts.messages; this.dependencies = opts.dependencies; } - get model(): string | undefined { - return this.definition.model; - } - - get outputJSONSchema(): PromptJsonSchema | undefined { - return this.definition.outputSchema?.toJSONSchema(); + [Symbol.iterator](): Iterator { + return this.messages + .map((message) => + attachDependenciesToMessage(message, this.dependencies.prompts), + ) + [Symbol.iterator](); } - get spanInfo() { - return { - metadata: { - prompt: this.dependencies, - }, - }; + to( + adapter: PromptAdapter, + ): Extendable { + return makeExtendableAdapterResult( + adapter({ + kind: "messages", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + messages: this.messages, + dependencies: this.dependencies, + }), + ); } +} - parseOutput(value: unknown): TOutput { - if (!this.definition.outputSchema) { - return value as TOutput; - } - return this.definition.outputSchema.parse(value, "output"); - } +class BuiltStringPrompt { + readonly [builtPromptMarker] = true; + readonly kind = "string"; + readonly definition: { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + }; + readonly input: TInput; + readonly content: string; + readonly dependencies: PromptDependencies; - asText(): string { - return this.messages - .map((message) => `${message.role}: ${message.content}`) - .join("\n"); + constructor(opts: BuiltStringPromptOptions) { + this.definition = opts.definition; + this.input = opts.input; + this.content = opts.content; + this.dependencies = opts.dependencies; } - to(adapter: PromptAdapter): TResult { - return adapter(this); + to( + adapter: PromptAdapter, + ): Extendable { + return makeExtendableAdapterResult( + adapter({ + kind: "string", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + content: this.content, + messages: [{ role: "user", content: this.content }], + dependencies: this.dependencies, + }), + ); } } -export type OpenAIChatPromptArgs = { - model?: string; - messages: PromptMessage[]; - response_format?: { - type: "json_schema"; - json_schema: { - name: string; - schema: PromptJsonSchema; - strict: true; - }; - }; - span_info: { - metadata: { - prompt: PromptDependencies; - }; - }; -}; +function makeExtendableAdapterResult( + result: TResult, +): Extendable { + if (!isMergeableObject(result)) { + throw new Error("prompt adapters must return an object"); + } -export type AISDKGenerateObjectPromptArgs = { - model?: string; - messages: PromptMessage[]; - schema?: PromptJsonSchema; - experimental_telemetry: { - metadata: { - braintrustPrompt: PromptDependencies; - }; - }; -}; + const extendable = result as Extendable; + Object.defineProperty(extendable, "extend", { + value: (extension: TExtension) => { + if (!isMergeableObject(extension)) { + throw new Error("extend must receive an object"); + } + return makeExtendableAdapterResult( + deepMergeObjects(extendable, extension) as DeepMerge< + TResult, + TExtension + >, + ); + }, + enumerable: false, + configurable: true, + }); + return extendable; +} -function openAIChatAdapter( - builtPrompt: BuiltPrompt, -): OpenAIChatPromptArgs { - const outputSchema = builtPrompt.outputJSONSchema; - return { - model: builtPrompt.model, - messages: builtPrompt.messages, - ...(outputSchema - ? { - response_format: { - type: "json_schema" as const, - json_schema: { - name: schemaName(builtPrompt.dependencies.root.slug), - schema: outputSchema, - strict: true as const, - }, - }, - } - : {}), - span_info: builtPrompt.spanInfo, - }; +function deepMergeObjects( + base: PromptExtension, + extension: PromptExtension, +): PromptExtension { + const merged: PromptExtension = { ...base }; + for (const [key, value] of Object.entries(extension)) { + const baseValue = merged[key]; + merged[key] = + isMergeableObject(baseValue) && isMergeableObject(value) + ? deepMergeObjects(baseValue, value) + : value; + } + return merged; } -function aiSDKGenerateObjectAdapter( - builtPrompt: BuiltPrompt, -): AISDKGenerateObjectPromptArgs { - return { - model: builtPrompt.model, - messages: builtPrompt.messages, - schema: builtPrompt.outputJSONSchema, - experimental_telemetry: { - metadata: { - braintrustPrompt: builtPrompt.dependencies, - }, - }, - }; +function isMergeableObject(value: unknown): value is PromptExtension { + return typeof value === "object" && value !== null && !Array.isArray(value); } +type AnyBuiltPrompt = + | BuiltMessagesPrompt + | BuiltStringPrompt; + function definePrompt< - TInputSchema extends PromptSchema, - TOutputSchema extends PromptSchema | undefined = undefined, + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined = undefined, + TRenderResult extends PromptRenderResult = PromptRenderResult, >( - opts: PromptDefinitionOptions, -): PromptDefinition { + opts: PromptDefinitionOptions, +): PromptDefinition { return new PromptDefinition(opts); } @@ -427,71 +689,140 @@ function messageTag(role: PromptRole) { return ( strings: TemplateStringsArray, ...values: readonly unknown[] - ): PromptMessage => ({ - role, - content: renderTaggedTemplate(strings, values), - }); + ): PromptMessage => { + const rendered = renderTaggedTemplate(strings, values); + return attachDependenciesToMessage( + { role, content: rendered.content }, + rendered.dependencies, + ); + }; +} + +function textTag( + strings: TemplateStringsArray, + ...values: readonly unknown[] +): PromptText { + const rendered = renderTaggedTemplate(strings, values); + return { + [promptTextMarker]: true, + content: rendered.content, + dependencies: rendered.dependencies, + }; } function renderTaggedTemplate( strings: TemplateStringsArray, values: readonly unknown[], -): string { - let rendered = strings[0] ?? ""; +): { content: string; dependencies: PromptDependencyEntry[] } { + let content = strings[0] ?? ""; + const dependencies: PromptDependencyEntry[] = []; for (let i = 0; i < values.length; i++) { - rendered += stringifyTemplateValue(values[i]) + (strings[i + 1] ?? ""); + const rendered = stringifyTemplateValue(values[i]); + content += rendered.content + (strings[i + 1] ?? ""); + dependencies.push(...rendered.dependencies); } - return rendered; + return { content, dependencies }; } -function stringifyTemplateValue(value: unknown): string { +function stringifyTemplateValue(value: unknown): { + content: string; + dependencies: PromptDependencyEntry[]; +} { if (value === undefined || value === null) { - return ""; + return { content: "", dependencies: [] }; } - if (isBuiltPrompt(value)) { - return value.asText(); + if (isBuiltStringPrompt(value)) { + return { content: value.content, dependencies: value.dependencies.prompts }; + } + if (isBuiltMessagesPrompt(value)) { + throw new Error("message prompts cannot be interpolated as text"); + } + if (isPromptText(value)) { + return { content: value.content, dependencies: value.dependencies }; } if (isPromptDefinition(value)) { - return `[prompt:${value.slug}${value.version ? `@${value.version}` : ""}]`; + return { + content: `[prompt:${value.slug}${value.version ? `@${value.version}` : ""}]`, + dependencies: [], + }; } if ( typeof value === "string" || typeof value === "number" || typeof value === "boolean" ) { - return String(value); + return { content: String(value), dependencies: [] }; } - return JSON.stringify(value); + return { content: JSON.stringify(value), dependencies: [] }; } -function flattenRenderable( - renderable: PromptRenderable | readonly PromptRenderable[], - parent: string, -): { messages: PromptMessage[]; dependencies: PromptDependencyEntry[] } { - const messages: PromptMessage[] = []; - const dependencies: PromptDependencyEntry[] = []; - const stack = Array.isArray(renderable) ? renderable : [renderable]; - - for (const item of stack) { - if (Array.isArray(item)) { - const flattened = flattenRenderable(item, parent); - messages.push(...flattened.messages); - dependencies.push(...flattened.dependencies); - } else if (isBuiltPrompt(item)) { - messages.push(...item.messages); - dependencies.push( - ...item.dependencies.prompts.map((entry) => ({ - ...entry, - role: "include" as const, - parent, - })), - ); - } else { - messages.push(item); - } +function attachDependenciesToMessage( + message: PromptMessage, + dependencies: PromptDependencyEntry[], +): PromptMessage { + if (dependencies.length === 0) { + return message; + } + const messageWithDependencies: PromptMessageWithDependencies = { ...message }; + Object.defineProperty(messageWithDependencies, promptDependencyMarker, { + value: dependencies, + enumerable: false, + }); + return messageWithDependencies; +} + +function assertPromptMessage(value: unknown, path: string): PromptMessage { + if ( + !isRecord(value) || + (value.role !== "system" && + value.role !== "user" && + value.role !== "assistant") || + typeof value.content !== "string" + ) { + throw new Error(`${path} must be a prompt message`); } + return value as PromptMessage; +} - return { messages, dependencies }; +function createPromptDependencies( + root: PromptDependencies["root"], + input: unknown, + entries: PromptDependencyEntry[], +): PromptDependencies { + return { + root, + prompts: [ + { + ...root, + role: "root", + input: sanitizeDependencyInput(input), + }, + ...dedupeDependencyEntries(entries), + ], + }; +} + +function collectDependencyEntries( + entries: readonly PromptDependencyEntry[], + parent: string, +): PromptDependencyEntry[] { + return entries.map((entry) => ({ + ...entry, + role: "include" as const, + parent, + })); +} + +function collectMessageDependencies( + messages: readonly PromptMessage[], + parent: string, +): PromptDependencyEntry[] { + return messages.flatMap((message) => + collectDependencyEntries( + (message as PromptMessageWithDependencies)[promptDependencyMarker] ?? [], + parent, + ), + ); } function collectBuiltPromptDependencies( @@ -499,11 +830,7 @@ function collectBuiltPromptDependencies( parent: string, ): PromptDependencyEntry[] { if (isBuiltPrompt(value)) { - return value.dependencies.prompts.map((entry) => ({ - ...entry, - role: "include" as const, - parent, - })); + return collectDependencyEntries(value.dependencies.prompts, parent); } if (Array.isArray(value)) { return value.flatMap((item) => @@ -535,7 +862,10 @@ function dedupeDependencyEntries( function sanitizeDependencyInput(value: unknown): unknown { if (isBuiltPrompt(value)) { return { - type: "built_prompt", + type: + value.kind === "messages" + ? "built_messages_prompt" + : "built_string_prompt", root: value.dependencies.root, }; } @@ -560,17 +890,24 @@ function sanitizeDependencyInput(value: unknown): unknown { return value; } -function schemaName(slug: string): string { - const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); - return name.length > 0 ? `${name}_output` : "prompt_output"; -} - -function isBuiltPrompt(value: unknown): value is BuiltPrompt { +function isBuiltPrompt(value: unknown): value is AnyBuiltPrompt { return ( typeof value === "object" && value !== null && builtPromptMarker in value ); } +function isBuiltMessagesPrompt( + value: unknown, +): value is BuiltMessagesPrompt { + return isBuiltPrompt(value) && value.kind === "messages"; +} + +function isBuiltStringPrompt( + value: unknown, +): value is BuiltStringPrompt { + return isBuiltPrompt(value) && value.kind === "string"; +} + function isPromptDefinition(value: unknown): value is AnyPromptDefinition { return ( typeof value === "object" && @@ -579,6 +916,12 @@ function isPromptDefinition(value: unknown): value is AnyPromptDefinition { ); } +function isPromptText(value: unknown): value is PromptText { + return ( + typeof value === "object" && value !== null && promptTextMarker in value + ); +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -597,12 +940,17 @@ function mergePromptInputs(parent: unknown, overrides: unknown): unknown { function buildAnyPrompt( definition: AnyPromptDefinition, input: unknown, -): BuiltPrompt { - return definition.build(input as never) as BuiltPrompt; +): AnyBuiltPrompt { + return definition.build(input as never) as AnyBuiltPrompt; } -function stringSchema(): PromptSchema { - return new PromptSchema( +function stringSchema(): PromptSchema< + string, + string, + unknown, + TDomain +> { + return new PromptSchema( (value, path) => { if (typeof value !== "string") { throw new Error(`${path} must be a string`); @@ -613,8 +961,13 @@ function stringSchema(): PromptSchema { ); } -function numberSchema(): PromptSchema { - return new PromptSchema( +function numberSchema(): PromptSchema< + number, + number, + unknown, + TDomain +> { + return new PromptSchema( (value, path) => { if (typeof value !== "number") { throw new Error(`${path} must be a number`); @@ -625,8 +978,13 @@ function numberSchema(): PromptSchema { ); } -function booleanSchema(): PromptSchema { - return new PromptSchema( +function booleanSchema(): PromptSchema< + boolean, + boolean, + unknown, + TDomain +> { + return new PromptSchema( (value, path) => { if (typeof value !== "boolean") { throw new Error(`${path} must be a boolean`); @@ -637,10 +995,13 @@ function booleanSchema(): PromptSchema { ); } -function enumSchema( +function enumSchema< + const TValues extends readonly [string, ...string[]], + TDomain extends SchemaDomain = "input", +>( values: TValues, -): PromptSchema { - return new PromptSchema( +): PromptSchema { + return new PromptSchema( (value, path) => { if (typeof value !== "string" || !values.includes(value)) { throw new Error(`${path} must be one of ${values.join(", ")}`); @@ -651,10 +1012,23 @@ function enumSchema( ); } -function arraySchema>( +function createArraySchema< + TItemSchema extends AnySchema, + TDomain extends SchemaDomain, +>( item: TItemSchema, -): PromptSchema[], InferInputSchema[]> { - return new PromptSchema( +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain +> { + return new PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain + >( (value, path, root) => { if (!Array.isArray(value)) { throw new Error(`${path} must be an array`); @@ -667,10 +1041,45 @@ function arraySchema>( ); } -function objectSchema( +function arraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "input" +> { + return createArraySchema(item); +} + +function outputArraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "output" +> { + return createArraySchema(item); +} + +function createObjectSchema< + TShape extends SchemaShape, + TDomain extends SchemaDomain, +>( shape: TShape, -): PromptSchema, InferInputObject> { - return new PromptSchema( +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain +> { + return new PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain + >( (value, path, root) => { if (typeof value !== "object" || value === null || Array.isArray(value)) { throw new Error(`${path} must be an object`); @@ -702,90 +1111,166 @@ function objectSchema( ); } -function unknownSchema(): PromptSchema { - return new PromptSchema( +function objectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "input" +> { + return createObjectSchema(shape); +} + +function outputObjectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "output" +> { + return createObjectSchema(shape); +} + +function unknownSchema(): PromptSchema< + unknown, + unknown, + unknown, + TDomain +> { + return new PromptSchema( (value) => value, () => ({}), ); } -function builtPromptSchema(): PromptSchema< - BuiltPrompt +function builtMessagesPromptSchema(): PromptSchema< + BuiltMessagesPrompt, + BuiltMessagesPrompt, + unknown, + "input" > { - return new PromptSchema( - (value, path) => { - if (!isBuiltPrompt(value)) { - throw new Error(`${path} must be a built prompt`); - } - return value as BuiltPrompt; - }, - () => ({ type: "object", "x-bt-type": "built_prompt" }), - ); + return builtPromptSchema("messages"); } -function promptDefinitionSchema< - TDefinition extends AnyPromptDefinition = AnyPromptDefinition, ->(): PromptSchema { +function builtStringPromptSchema(): PromptSchema< + BuiltStringPrompt, + BuiltStringPrompt, + unknown, + "input" +> { + return builtPromptSchema("string"); +} + +function messagesPromptDefinitionSchema< + TDefinition extends AnyMessagesPromptDefinition, +>( + definition: TDefinition, +): PromptSchema< + BuiltMessagesPrompt, OutputOf>, + PromptInputValue, + PromptFieldKind, + "input" +> { + return promptDefinitionSchema("messages", definition); +} + +function stringPromptDefinitionSchema< + TDefinition extends AnyStringPromptDefinition, +>( + definition: TDefinition, +): PromptSchema< + BuiltStringPrompt, OutputOf>, + PromptInputValue, + PromptFieldKind, + "input" +> { + return promptDefinitionSchema("string", definition); +} + +function builtPromptSchema( + kind: TKind, +): PromptSchema< + BuiltPromptForKind, + BuiltPromptForKind, + unknown, + "input" +> { + const label = kind === "messages" ? "messages" : "string"; return new PromptSchema( (value, path) => { - if (!isPromptDefinition(value)) { - throw new Error(`${path} must be a prompt definition`); + if (isBuiltPrompt(value)) { + if (value.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return value as BuiltPromptForKind; } - return value as TDefinition; + + throw new Error(`${path} must be a built ${label} prompt`); }, - () => ({ type: "object", "x-bt-type": "prompt_definition" }), + () => ({ + type: "object", + "x-bt-type": + kind === "messages" ? "built_messages_prompt" : "built_string_prompt", + }), ); } -function promptSchema( +function promptDefinitionSchema< + TDefinition extends AnyPromptDefinition, + TKind extends PromptKind, +>( + kind: TKind, definition: TDefinition, ): PromptSchema< - BuiltPrompt, OutputOf>, - PromptInputValue ->; -function promptSchema(): PromptSchema< - BuiltPrompt, - DynamicPromptInputValue ->; -function promptSchema( - definition?: AnyPromptDefinition, -): PromptSchema, unknown> { + BuiltPromptForKind, OutputOf, TKind>, + PromptInputValue, + PromptFieldKind, + "input" +> { + const label = kind === "messages" ? "messages" : "string"; return new PromptSchema( (value, path, root) => { if (isBuiltPrompt(value)) { - return value; + if (value.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return value as BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TKind + >; } - let promptDefinition = definition; - let promptInput = value; - if (!promptDefinition) { - if (isPromptDefinition(value)) { - promptDefinition = value; - promptInput = undefined; - } else if (isRecord(value) && isPromptDefinition(value.prompt)) { - promptDefinition = value.prompt; - promptInput = value.input; - } else { - throw new Error(`${path} must be a prompt or built prompt`); - } - } else if (isPromptDefinition(value)) { - promptDefinition = value; - promptInput = undefined; - } else if (isRecord(value) && isPromptDefinition(value.prompt)) { - promptDefinition = value.prompt; - promptInput = value.input; + if ( + isPromptDefinition(value) || + (isRecord(value) && isPromptDefinition(value.prompt)) + ) { + throw new Error( + `${path} must be a built ${label} prompt or prompt input`, + ); } - return buildAnyPrompt( - promptDefinition, - mergePromptInputs(root, promptInput), - ); + const built = buildAnyPrompt(definition, mergePromptInputs(root, value)); + if (built.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return built as BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TKind + >; }, - () => ({ type: "object", "x-bt-type": "prompt" }), + () => ({ + type: "object", + "x-bt-type": + kind === "messages" ? "built_messages_prompt" : "built_string_prompt", + }), ); } -export const s = { +const inputSchemaHelpers = { string: stringSchema, number: numberSchema, boolean: booleanSchema, @@ -793,12 +1278,22 @@ export const s = { array: arraySchema, object: objectSchema, unknown: unknownSchema, - // Requires an already-built prompt value; useful when the caller owns construction. - builtPrompt: builtPromptSchema, - // Accepts a built prompt, or builds a prompt definition by merging parent input plus overrides. - prompt: promptSchema, - // Accepts an unbuilt prompt definition as data; render code decides whether/when to build it. - promptDefinition: promptDefinitionSchema, + builtMessagesPrompt: builtMessagesPromptSchema, + builtStringPrompt: builtStringPromptSchema, + messagesPromptDefinition: messagesPromptDefinitionSchema, + stringPromptDefinition: stringPromptDefinitionSchema, +}; + +const outputSchemaHelpers = { + string: () => stringSchema<"output">(), + number: () => numberSchema<"output">(), + boolean: () => booleanSchema<"output">(), + enum: ( + values: TValues, + ) => enumSchema(values), + array: outputArraySchema, + object: outputObjectSchema, + unknown: () => unknownSchema<"output">(), }; export const prompt = { @@ -806,11 +1301,8 @@ export const prompt = { system: messageTag("system"), user: messageTag("user"), assistant: messageTag("assistant"), - asText: (builtPrompt: BuiltPrompt) => builtPrompt.asText(), + text: textTag, isBuiltPrompt, isPromptDefinition, - adapters: { - openAIChat: openAIChatAdapter, - aiSDKGenerateObject: aiSDKGenerateObjectAdapter, - }, + adapters, }; diff --git a/js/src/exports.ts b/js/src/exports.ts index e1b72f7c0..12e469ca1 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -262,7 +262,7 @@ export { PromptDefinitionWithTools, } from "./prompt-schemas"; -export { prompt, s } from "./experimental-prompt-api"; +export { prompt } from "./experimental-prompt-api"; export type { Trace, SpanData, GetThreadOptions } from "./trace"; export { SpanFetcher, CachedSpanFetcher, LocalTrace } from "./trace"; From 68fc2fc2b06f9b2f41e9c5285dc20f7eb38976f9 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Jul 2026 15:38:42 +0200 Subject: [PATCH 3/6] mustache stuff (lists and inlining) --- js/src/experimental-prompt-api.test.ts | 329 ++++++++++++- js/src/experimental-prompt-api.ts | 645 ++++++++++++++++++++++++- knip.jsonc | 1 + 3 files changed, 942 insertions(+), 33 deletions(-) diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts index 0887a63b2..6cb197a96 100644 --- a/js/src/experimental-prompt-api.test.ts +++ b/js/src/experimental-prompt-api.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "vitest"; -import { prompt } from "./experimental-prompt-api"; +import { prompt, promptDefinitionToMustache } from "./experimental-prompt-api"; describe("experimental prompt API", () => { test("builds message prompts and translates to OpenAI chat args", () => { @@ -16,9 +16,9 @@ describe("experimental prompt API", () => { body: s.string(), urgency: s.enum(["low", "medium", "high"]), }), - render: ({ input }) => [ + render: ({ variables }) => [ prompt.system`You write concise support replies.`, - prompt.user`Ticket: ${input.ticket}`, + prompt.user`Ticket: ${variables.ticket}`, ], }); @@ -84,7 +84,7 @@ describe("experimental prompt API", () => { s.object({ policy: s.string(), }), - render: ({ input }) => prompt.text`Policy: ${input.policy}`, + render: ({ variables }) => prompt.text`Policy: ${variables.policy}`, }); const built = policyText.build({ policy: "Prefer short answers." }); @@ -123,6 +123,280 @@ describe("experimental prompt API", () => { } }); + test("exports message prompt data as inlined mustache templates", () => { + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + input: (s) => + s.object({ + customer: s.object({ + name: s.string(), + }), + ticket: s.string(), + }), + output: (s) => + s.object({ + body: s.string(), + }), + render: ({ variables }) => [ + prompt.system`You write concise support replies.`, + prompt.user`Customer: ${variables.customer.name}\nTicket: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data).toMatchObject({ + slug: "support-reply", + model: "gpt-4o", + kind: "messages", + inputSchema: { + type: "object", + required: ["customer", "ticket"], + }, + outputSchema: { + type: "object", + required: ["body"], + }, + messages: [ + { role: "system", content: "You write concise support replies." }, + { + role: "user", + content: "Customer: {{customer.name}}\nTicket: {{ticket}}", + }, + ], + }); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You write concise support replies." }, + { + role: "user", + content: "Customer: {{customer.name}}\nTicket: {{ticket}}", + }, + ], + }); + }); + + test("exports string prompt data as a mustache user message template", () => { + const summarize = prompt.define({ + slug: "summarize", + model: "gpt-4o-mini", + input: (s) => + s.object({ + text: s.string(), + }), + render: ({ variables }) => prompt.text`Summarize: ${variables.text}`, + }); + + const data = summarize.toPromptData(); + + expect(data).toMatchObject({ + slug: "summarize", + model: "gpt-4o-mini", + kind: "string", + content: "Summarize: {{text}}", + }); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Summarize: {{text}}" }], + }); + }); + + test("inlines nested message prompt definitions in mustache prompt data", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + version: "v3", + input: (s) => + s.object({ + company: s.string(), + tone: s.string(), + }), + render: ({ variables }) => [ + prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, + ], + }); + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + input: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + voice: s.messagesPromptDefinition(brandVoice), + }), + render: ({ variables }) => [ + ...variables.voice, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "system", + content: "Use {{company}}'s {{voice.tone}} voice.", + }, + { role: "user", content: "Draft a reply for: {{ticket}}" }, + ]); + expect(data.dependencies.prompts).toMatchObject([ + { slug: "support-reply", role: "root" }, + { + slug: "brand-voice", + version: "v3", + role: "include", + parent: "support-reply", + }, + ]); + }); + + test("inlines nested string prompt definitions in mustache prompt data", () => { + const policyText = prompt.define({ + slug: "policy-text", + version: "v2", + input: (s) => + s.object({ + company: s.string(), + text: s.string(), + }), + render: ({ variables }) => + prompt.text`${variables.company}: ${variables.text}`, + }); + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + input: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + policy: s.stringPromptDefinition(policyText), + }), + render: ({ variables }) => [ + prompt.system`Follow this policy: ${variables.policy}`, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "system", + content: "Follow this policy: {{company}}: {{policy.text}}", + }, + { role: "user", content: "Draft a reply for: {{ticket}}" }, + ]); + expect(data.dependencies.prompts).toMatchObject([ + { slug: "support-reply", role: "root" }, + { + slug: "policy-text", + version: "v2", + role: "include", + parent: "support-reply", + }, + ]); + }); + + test("promptDefinitionToMustache requires a model", () => { + const modeless = prompt.define({ + slug: "modeless", + input: (s) => s.object({ text: s.string() }), + render: ({ variables }) => [prompt.user`Say ${variables.text}`], + }); + + expect(() => promptDefinitionToMustache(modeless.toPromptData())).toThrow( + "Cannot convert prompt data to mustache without a model", + ); + }); + + test("exports array list templates as mustache sections", () => { + const itemList = prompt.define({ + slug: "item-list", + model: "gpt-4o", + input: (s) => + s.object({ + items: s.array( + s.object({ + foobar: s.string(), + author: s.object({ + name: s.string(), + }), + }), + ), + }), + render: ({ variables }) => [ + prompt.user`Items:\n${variables.items.list`- ${variables.items.list.foobar} by ${variables.items.list.author.name}\n`}`, + ], + }); + + const built = itemList.build({ + items: [ + { foobar: "first", author: { name: "Ada" } }, + { foobar: "second", author: { name: "Grace" } }, + ], + }); + const data = itemList.toPromptData(); + + expect(built.messages).toEqual([ + { + role: "user", + content: "Items:\n- first by Ada\n- second by Grace\n", + }, + ]); + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "user", + content: + "Items:\n{{#items}}- {{foobar}} by {{author.name}}\n{{/items}}", + }, + ]); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o", + messages: [ + { + role: "user", + content: + "Items:\n{{#items}}- {{foobar}} by {{author.name}}\n{{/items}}", + }, + ], + }); + }); + + test("passes parsed runtime values separately from template variables", () => { + const itemSummary = prompt.define({ + slug: "item-summary", + input: (s) => + s.object({ + items: s.array( + s.object({ + foobar: s.string(), + }), + ), + }), + render: ({ values }) => [ + prompt.user`Second: ${values.items[1]?.foobar}\nAll: ${values.items.map((item) => item.foobar).join(", ")}`, + ], + }); + + expect( + itemSummary.build({ + items: [{ foobar: "first" }, { foobar: "second" }], + }).messages, + ).toEqual([ + { + role: "user", + content: "Second: second\nAll: first, second", + }, + ]); + expect(() => itemSummary.toPromptData()).toThrow( + "Runtime values are not available while exporting prompt data; use variables in prompt templates.", + ); + }); + test("extends adapter args with a typed deep merge", () => { const classify = prompt.define({ slug: "classify", @@ -135,7 +409,7 @@ describe("experimental prompt API", () => { s.object({ label: s.enum(["bug", "question"]), }), - render: ({ input }) => [prompt.user`Classify: ${input.text}`], + render: ({ variables }) => [prompt.user`Classify: ${variables.text}`], }); const built = classify.build({ text: "It crashes" }); @@ -226,8 +500,8 @@ describe("experimental prompt API", () => { company: s.string(), tone: s.string(), }), - render: ({ input }) => [ - prompt.system`Use ${input.company}'s ${input.tone} voice.`, + render: ({ variables }) => [ + prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, ], }); @@ -240,9 +514,9 @@ describe("experimental prompt API", () => { ticket: s.string(), voice: s.messagesPromptDefinition(brandVoice), }), - render: ({ input }) => [ - ...input.voice, - prompt.user`Draft a reply for: ${input.ticket}`, + render: ({ variables }) => [ + ...variables.voice, + prompt.user`Draft a reply for: ${variables.ticket}`, ], }); @@ -334,7 +608,8 @@ describe("experimental prompt API", () => { company: s.string(), policy: s.string(), }), - render: ({ input }) => prompt.text`${input.company}: ${input.policy}`, + render: ({ variables }) => + prompt.text`${variables.company}: ${variables.policy}`, }); const supportReply = prompt.define({ @@ -345,9 +620,9 @@ describe("experimental prompt API", () => { ticket: s.string(), policy: s.stringPromptDefinition(policyText), }), - render: ({ input }) => [ - prompt.system`Follow this policy: ${input.policy}`, - prompt.user`Draft a reply for: ${input.ticket}`, + render: ({ variables }) => [ + prompt.system`Follow this policy: ${variables.policy}`, + prompt.user`Draft a reply for: ${variables.ticket}`, ], }); @@ -410,7 +685,9 @@ describe("experimental prompt API", () => { s.object({ topic: s.string(), }), - render: ({ input }) => [prompt.user`Message about ${input.topic}`], + render: ({ variables }) => [ + prompt.user`Message about ${variables.topic}`, + ], }); const stringPrompt = prompt.define({ slug: "string-prompt", @@ -418,7 +695,7 @@ describe("experimental prompt API", () => { s.object({ topic: s.string(), }), - render: ({ input }) => prompt.text`String about ${input.topic}`, + render: ({ variables }) => prompt.text`String about ${variables.topic}`, }); const consumeBoth = prompt.define({ @@ -428,9 +705,9 @@ describe("experimental prompt API", () => { messagePart: s.builtMessagesPrompt(), stringPart: s.builtStringPrompt(), }), - render: ({ input }) => [ - ...input.messagePart, - prompt.user`Fragment: ${input.stringPart}`, + render: ({ variables }) => [ + ...variables.messagePart, + prompt.user`Fragment: ${variables.stringPart}`, ], }); @@ -487,7 +764,9 @@ describe("experimental prompt API", () => { s.object({ topic: s.string(), }), - render: ({ input }) => [prompt.user`Message about ${input.topic}`], + render: ({ variables }) => [ + prompt.user`Message about ${variables.topic}`, + ], }); const stringPrompt = prompt.define({ slug: "string-prompt", @@ -495,7 +774,7 @@ describe("experimental prompt API", () => { s.object({ topic: s.string(), }), - render: ({ input }) => prompt.text`String about ${input.topic}`, + render: ({ variables }) => prompt.text`String about ${variables.topic}`, }); const messagePart = messagePrompt.build({ topic: "tracing" }); @@ -526,7 +805,7 @@ describe("experimental prompt API", () => { s.object({ ok: s.boolean(), }), - render: ({ input }) => [prompt.user`Count: ${input.count}`], + render: ({ variables }) => [prompt.user`Count: ${variables.count}`], }); const invalidRenderPrompt = prompt.define({ slug: "invalid-render", @@ -565,7 +844,7 @@ describe("experimental prompt API", () => { ok: s.boolean(), }); }, - render: ({ input }) => [prompt.user`Topic: ${input.topic}`], + render: ({ variables }) => [prompt.user`Topic: ${variables.topic}`], }); expect(inputHelperKeys).toContain("builtMessagesPrompt"); @@ -642,7 +921,7 @@ describe("experimental prompt API", () => { s.object({ label: s.enum(["bug", "question"]), }), - render: ({ input }) => [prompt.user`Classify: ${input.text}`], + render: ({ variables }) => [prompt.user`Classify: ${variables.text}`], }); const summarize = prompt.define({ slug: "summarize", @@ -650,7 +929,7 @@ describe("experimental prompt API", () => { s.object({ text: s.string(), }), - render: ({ input }) => prompt.text`Summarize: ${input.text}`, + render: ({ variables }) => prompt.text`Summarize: ${variables.text}`, }); expect( diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts index c2e23baf2..575e25c3b 100644 --- a/js/src/experimental-prompt-api.ts +++ b/js/src/experimental-prompt-api.ts @@ -1,4 +1,5 @@ import { adapters } from "./experimental-prompt-adapters"; +import type { PromptDefinition as MustachePromptDefinition } from "./prompt-schemas"; type JsonPrimitive = string | number | boolean | null; @@ -16,6 +17,22 @@ export type PromptJsonSchema = { type SchemaParser = (value: unknown, path: string, root: unknown) => T; type SchemaDomain = "input" | "output"; +type PromptKind = "messages" | "string"; + +type PromptSchemaTemplateInfo = + | { + type: "object"; + shape: SchemaShape; + } + | { + type: "array"; + item: AnySchema; + } + | { + type: "promptDefinition"; + definition: AnyPromptDefinition; + kind: PromptKind; + }; class PromptSchema< TParsed, @@ -32,6 +49,7 @@ class PromptSchema< private readonly parser: SchemaParser, private readonly jsonSchema: () => PromptJsonSchema, public readonly isOptional = false, + public readonly templateInfo?: PromptSchemaTemplateInfo, ) {} parse(value: unknown, path = "value", root: unknown = value): TParsed { @@ -58,6 +76,7 @@ class PromptSchema< value === undefined ? undefined : this.parser(value, path, root), () => this.jsonSchema(), true, + this.templateInfo, ); } } @@ -141,10 +160,14 @@ const promptTextMarker = Symbol("braintrust.experimental_prompt.text"); const promptDependencyMarker = Symbol( "braintrust.experimental_prompt.dependencies", ); +const mustacheTemplateValueMarker = Symbol( + "braintrust.experimental_prompt.mustache_template_value", +); +const templateValueStateMarker = Symbol( + "braintrust.experimental_prompt.template_value_state", +); type PromptRole = "system" | "user" | "assistant"; -type PromptKind = "messages" | "string"; - export type PromptMessage = { role: PromptRole; content: string; @@ -160,6 +183,21 @@ type PromptText = { readonly dependencies: PromptDependencyEntry[]; }; +type PromptVariableMode = "runtime" | "mustache"; + +type TemplateValueState = { + readonly path: string; + readonly mode: PromptVariableMode; + readonly runtimeValue?: unknown; + readonly sectionPath?: string; + readonly relativePath?: string; +}; + +type MustacheTemplateValue = { + readonly [mustacheTemplateValueMarker]: true; + readonly [templateValueStateMarker]: TemplateValueState; +}; + type PromptDependencyEntry = { id?: string; slug: string; @@ -180,16 +218,67 @@ export type PromptDependencies = { prompts: PromptDependencyEntry[]; }; +export type ExperimentalPromptData = { + id?: string; + slug: string; + name?: string; + version?: string; + model?: string; + inputSchema: PromptJsonSchema; + outputSchema?: PromptJsonSchema; + dependencies: PromptDependencies; +} & ( + | { + kind: "messages"; + messages: PromptMessage[]; + } + | { + kind: "string"; + content: string; + } +); + type PromptRenderResult = readonly PromptMessage[] | PromptText; -type PromptRenderContext = { - input: TInput; +type PromptRenderContext = { + variables: TVariables; + values: TValues; include: ( definition: TDefinition, input: InputOf, ) => BuiltPromptOf; }; +type PromptTemplateScope = { + rootKeys: ReadonlySet; + pathForKey: (key: string) => string; +}; + +type TemplateNestedPromptBuilder = ( + definition: AnyPromptDefinition, + kind: PromptKind, + fieldPath: string, +) => AnyBuiltPrompt; + +type PromptListTag = ( + strings: TemplateStringsArray, + ...values: readonly unknown[] +) => PromptText; + +type PromptTemplateField = TValue extends AnyBuiltPrompt + ? TValue + : unknown & + (TValue extends readonly (infer TItem)[] + ? { list: PromptListTag & PromptTemplateField } + : TValue extends object + ? { [K in keyof TValue]: PromptTemplateField } + : {}); + +type TemplateRenderContext = { + sectionPath?: string; + item?: unknown; +}; + type InputSchemaHelpers = typeof inputSchemaHelpers; type OutputSchemaHelpers = typeof outputSchemaHelpers; @@ -206,7 +295,10 @@ type PromptDefinitionOptions< input: (s: InputSchemaHelpers) => TInputSchema; output?: (s: OutputSchemaHelpers) => TOutputSchema; render: ( - context: PromptRenderContext>, + context: PromptRenderContext< + PromptTemplateField>, + InferSchema + >, ) => TRenderResult; }; @@ -261,8 +353,18 @@ class PromptDefinition< "input", input, ) as InferSchema; + const variables = createPromptVariables( + this.inputSchema, + createRootTemplateScope(this.inputSchema), + "runtime", + parsedInput, + () => { + throw new Error("prompt variables could not resolve a built prompt"); + }, + ) as PromptTemplateField>; const rendered = this.renderer({ - input: parsedInput, + variables, + values: parsedInput, include: (definition, includeInput) => definition.build(includeInput) as BuiltPromptOf, }); @@ -347,6 +449,85 @@ class PromptDefinition< TRenderResult >; } + + toPromptData(): ExperimentalPromptData { + return this.compileTemplate(createRootTemplateScope(this.inputSchema)); + } + + private compileTemplate(scope: PromptTemplateScope): ExperimentalPromptData { + const variables = createPromptVariables( + this.inputSchema, + scope, + "mustache", + undefined, + (definition, kind, fieldPath) => { + const nested = definition.compileTemplate( + createNestedTemplateScope( + definition.inputSchema, + scope.rootKeys, + fieldPath, + ), + ); + return templateDataToBuiltPrompt(nested, kind); + }, + ) as PromptTemplateField>; + const rendered = this.renderer({ + variables, + values: createUnavailableValuesProxy() as InferSchema, + include: (definition) => { + const nested = definition.compileTemplate( + createRootTemplateScope(definition.inputSchema), + ); + return templateDataToBuiltPrompt(nested, nested.kind) as BuiltPromptOf< + typeof definition + >; + }, + }); + const root = promptDefinitionRoot(this); + const inputSnapshot = createTemplateDependencyInput( + this.inputSchema, + scope, + ); + + if (isPromptText(rendered)) { + const dependencies = createPromptDependencies(root, inputSnapshot, [ + ...collectBuiltPromptDependencies(variables, this.slug), + ...collectDependencyEntries(rendered.dependencies, this.slug), + ]); + + return { + ...root, + model: this.model, + inputSchema: this.inputSchema.toJSONSchema(), + outputSchema: this.outputSchema?.toJSONSchema(), + dependencies, + kind: "string", + content: rendered.content, + }; + } + + if (!Array.isArray(rendered)) { + throw new Error("render must return a message array or prompt.text"); + } + + const messages = rendered.map((message, index) => + assertPromptMessage(message, `render[${index}]`), + ); + const dependencies = createPromptDependencies(root, inputSnapshot, [ + ...collectBuiltPromptDependencies(variables, this.slug), + ...collectMessageDependencies(messages, this.slug), + ]); + + return { + ...root, + model: this.model, + inputSchema: this.inputSchema.toJSONSchema(), + outputSchema: this.outputSchema?.toJSONSchema(), + dependencies, + kind: "messages", + messages, + }; + } } type AnyPromptDefinition = PromptDefinition< @@ -713,24 +894,53 @@ function textTag( function renderTaggedTemplate( strings: TemplateStringsArray, values: readonly unknown[], + context?: TemplateRenderContext, ): { content: string; dependencies: PromptDependencyEntry[] } { let content = strings[0] ?? ""; const dependencies: PromptDependencyEntry[] = []; for (let i = 0; i < values.length; i++) { - const rendered = stringifyTemplateValue(values[i]); + const rendered = stringifyTemplateValue(values[i], context); content += rendered.content + (strings[i + 1] ?? ""); dependencies.push(...rendered.dependencies); } return { content, dependencies }; } -function stringifyTemplateValue(value: unknown): { +function stringifyTemplateValue( + value: unknown, + context?: TemplateRenderContext, +): { + content: string; + dependencies: PromptDependencyEntry[]; +}; +function stringifyTemplateValue( + value: unknown, + context?: TemplateRenderContext, +): { content: string; dependencies: PromptDependencyEntry[]; } { if (value === undefined || value === null) { return { content: "", dependencies: [] }; } + if (isMustacheTemplateValue(value)) { + const state = value[templateValueStateMarker]; + if (state.mode === "mustache") { + if (context?.sectionPath && state.sectionPath === context.sectionPath) { + return { + content: `{{${state.relativePath ?? "."}}}`, + dependencies: [], + }; + } + return { content: `{{${state.path}}}`, dependencies: [] }; + } + + const runtimeValue = + context?.sectionPath && state.sectionPath === context.sectionPath + ? getPathValue(context.item, state.relativePath) + : state.runtimeValue; + return { content: stringifyRuntimeValue(runtimeValue), dependencies: [] }; + } if (isBuiltStringPrompt(value)) { return { content: value.content, dependencies: value.dependencies.prompts }; } @@ -784,6 +994,395 @@ function assertPromptMessage(value: unknown, path: string): PromptMessage { return value as PromptMessage; } +function createRootTemplateScope( + inputSchema: InputSchema, +): PromptTemplateScope { + return { + rootKeys: getObjectSchemaKeys(inputSchema), + pathForKey: (key) => key, + }; +} + +function createNestedTemplateScope( + inputSchema: InputSchema, + rootKeys: ReadonlySet, + fieldPath: string, +): PromptTemplateScope { + const nestedKeys = getObjectSchemaKeys(inputSchema); + const fieldKey = lastPathSegment(fieldPath); + return { + rootKeys: new Set([...rootKeys, ...nestedKeys]), + pathForKey: (key) => + rootKeys.has(key) && key !== fieldKey ? key : `${fieldPath}.${key}`, + }; +} + +function getObjectSchemaKeys(schema: InputSchema): Set { + return schema.templateInfo?.type === "object" + ? new Set(Object.keys(schema.templateInfo.shape)) + : new Set(); +} + +function lastPathSegment(path: string): string { + return path.split(".").at(-1) ?? path; +} + +function createPromptVariables( + schema: InputSchema, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + path = "input", + sectionPath?: string, + relativePath?: string, +): unknown { + if (mode === "runtime" && isBuiltPrompt(runtimeValue)) { + return runtimeValue; + } + + const templateInfo = schema.templateInfo; + if (templateInfo?.type === "object") { + return createPromptVariableObject( + templateInfo.shape, + scope, + mode, + runtimeValue, + nestedPromptBuilder, + path === "input" ? undefined : path, + sectionPath, + relativePath, + ); + } + + if (templateInfo?.type === "array") { + return createPromptVariableArray( + templateInfo.item as InputSchema, + scope, + mode, + runtimeValue, + nestedPromptBuilder, + path, + sectionPath, + relativePath, + ); + } + + if (templateInfo?.type === "promptDefinition") { + return mode === "runtime" + ? runtimeValue + : nestedPromptBuilder(templateInfo.definition, templateInfo.kind, path); + } + + return createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }); +} + +function createPromptVariableObject( + shape: SchemaShape, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + basePath?: string, + sectionPath?: string, + relativePath?: string, +): Record { + const variableObject: Record = {}; + if (basePath) { + attachPromptVariableValue(variableObject, { + path: basePath, + mode, + runtimeValue, + sectionPath, + relativePath, + }); + } + for (const [key, schema] of Object.entries(shape)) { + const path = basePath ? `${basePath}.${key}` : scope.pathForKey(key); + const childRelativePath = sectionPath + ? relativePath + ? `${relativePath}.${key}` + : key + : undefined; + variableObject[key] = createPromptVariables( + schema as InputSchema, + scope, + mode, + getObjectProperty(runtimeValue, key), + nestedPromptBuilder, + path, + sectionPath, + childRelativePath, + ); + } + return variableObject; +} + +function createPromptVariableArray( + itemSchema: InputSchema, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + path: string, + sectionPath?: string, + relativePath?: string, +): Record { + const variableArray = createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }) as Record; + const listSectionPath = path; + const listTag = (( + strings: TemplateStringsArray, + ...values: readonly unknown[] + ): PromptText => { + if (mode === "mustache") { + const rendered = renderTaggedTemplate(strings, values, { + sectionPath: listSectionPath, + }); + const sectionName = relativePath ?? path; + return { + [promptTextMarker]: true, + content: `{{#${sectionName}}}${rendered.content}{{/${sectionName}}}`, + dependencies: rendered.dependencies, + }; + } + + const items = Array.isArray(runtimeValue) ? runtimeValue : []; + const renderedItems = items.map((item) => + renderTaggedTemplate(strings, values, { + sectionPath: listSectionPath, + item, + }), + ); + return { + [promptTextMarker]: true, + content: renderedItems.map((item) => item.content).join(""), + dependencies: renderedItems.flatMap((item) => item.dependencies), + }; + }) as PromptListTag & Record; + + attachPromptVariableValue(listTag, { + path, + mode, + runtimeValue, + sectionPath: listSectionPath, + relativePath: undefined, + }); + const itemVariables = createPromptVariables( + itemSchema, + scope, + mode, + undefined, + nestedPromptBuilder, + path, + listSectionPath, + ); + if (isRecord(itemVariables)) { + for (const key of Object.keys(itemVariables)) { + Object.defineProperty( + listTag, + key, + Object.getOwnPropertyDescriptor(itemVariables, key) ?? { + value: itemVariables[key], + enumerable: true, + }, + ); + } + } + + Object.defineProperty(variableArray, "list", { + value: listTag, + enumerable: true, + }); + return variableArray; +} + +function createTemplateDependencyInput( + schema: InputSchema, + scope: PromptTemplateScope, + path = "input", +): unknown { + const templateInfo = schema.templateInfo; + if (templateInfo?.type === "object") { + return Object.fromEntries( + Object.entries(templateInfo.shape).map(([key, childSchema]) => { + const childPath = + path === "input" ? scope.pathForKey(key) : `${path}.${key}`; + return [ + key, + createTemplateDependencyInput( + childSchema as InputSchema, + scope, + childPath, + ), + ]; + }), + ); + } + + if (templateInfo?.type === "array") { + return `{{${path}}}`; + } + + if (templateInfo?.type === "promptDefinition") { + return { + type: + templateInfo.kind === "messages" + ? "template_messages_prompt" + : "template_string_prompt", + root: promptDefinitionRoot(templateInfo.definition), + }; + } + + return `{{${path}}}`; +} + +function createPromptVariableValue( + state: TemplateValueState, +): MustacheTemplateValue { + return attachPromptVariableValue({}, state) as MustacheTemplateValue; +} + +function attachPromptVariableValue( + value: T, + state: TemplateValueState, +): T { + Object.defineProperties(value, { + [mustacheTemplateValueMarker]: { + value: true, + enumerable: false, + }, + [templateValueStateMarker]: { + value: state, + enumerable: false, + }, + toString: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + valueOf: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + [Symbol.toPrimitive]: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + }); + return value; +} + +function getObjectProperty(value: unknown, key: string): unknown { + if (isRecord(value)) { + return value[key]; + } + if (Array.isArray(value)) { + return value[Number(key)]; + } + return undefined; +} + +function getPathValue(value: unknown, path?: string): unknown { + if (!path) { + return value; + } + return path + .split(".") + .reduce((current, key) => getObjectProperty(current, key), value); +} + +function stringifyRuntimeValue(value: unknown): string { + if (value === undefined || value === null) { + return ""; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return String(value); + } + return JSON.stringify(value); +} + +function createUnavailableValuesProxy(): unknown { + return new Proxy( + {}, + { + get(_target, key) { + if (typeof key === "symbol") { + return undefined; + } + throw new Error( + "Runtime values are not available while exporting prompt data; use variables in prompt templates.", + ); + }, + }, + ); +} + +function isMustacheTemplateValue( + value: unknown, +): value is MustacheTemplateValue { + return ( + typeof value === "object" && + value !== null && + mustacheTemplateValueMarker in value + ); +} + +function promptDefinitionRoot( + definition: AnyPromptDefinition, +): PromptDependencies["root"] { + return { + id: definition.id, + slug: definition.slug, + name: definition.name, + version: definition.version, + }; +} + +function templateDataToBuiltPrompt( + data: ExperimentalPromptData, + kind: PromptKind, +): AnyBuiltPrompt { + if (data.kind !== kind) { + const label = kind === "messages" ? "messages" : "string"; + throw new Error(`template prompt must be a ${label} prompt`); + } + + const definition = { + model: data.model, + inputSchema: unknownSchema(), + outputSchema: undefined, + }; + if (data.kind === "messages") { + return new BuiltMessagesPrompt({ + definition, + input: data.dependencies.prompts[0]?.input, + messages: data.messages, + dependencies: data.dependencies, + }); + } + return new BuiltStringPrompt({ + definition, + input: data.dependencies.prompts[0]?.input, + content: data.content, + dependencies: data.dependencies, + }); +} + function createPromptDependencies( root: PromptDependencies["root"], input: unknown, @@ -1038,6 +1637,8 @@ function createArraySchema< ) as InferSchema[]; }, () => ({ type: "array", items: item.toJSONSchema() }), + false, + { type: "array", item }, ); } @@ -1108,6 +1709,8 @@ function createObjectSchema< .map(([key]) => key), additionalProperties: false, }), + false, + { type: "object", shape }, ); } @@ -1267,9 +1870,35 @@ function promptDefinitionSchema< "x-bt-type": kind === "messages" ? "built_messages_prompt" : "built_string_prompt", }), + false, + { type: "promptDefinition", definition, kind }, ); } +/** + * @internal Converts experimental prompt template data into the existing prompt + * definition shape. This is intended for future backend-saving code paths. + */ +export function promptDefinitionToMustache( + data: ExperimentalPromptData, +): MustachePromptDefinition { + if (!data.model) { + throw new Error("Cannot convert prompt data to mustache without a model"); + } + + if (data.kind === "messages") { + return { + model: data.model, + messages: data.messages, + }; + } + + return { + model: data.model, + messages: [{ role: "user", content: data.content }], + }; +} + const inputSchemaHelpers = { string: stringSchema, number: numberSchema, diff --git a/knip.jsonc b/knip.jsonc index 99bf84691..3775d1955 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -44,6 +44,7 @@ "ignoreDependencies": ["openai", "openai-v4", "openai-v5"], }, "js": { + "ignore": ["promptDefinitionToMustache"], "entry": [ "src/auto-instrumentations/bundler/*.ts", "tests/**/*.{ts,tsx,mts,cts,cjs,mjs}", From a98f5a7009d51b95254896c69808e3a993dba963 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Jul 2026 16:31:35 +0200 Subject: [PATCH 4/6] Support attachments --- js/src/experimental-prompt-adapters.ts | 348 ++++++++++++++++- js/src/experimental-prompt-api.test.ts | 281 +++++++++++--- js/src/experimental-prompt-api.ts | 501 ++++++++++++++++++++++--- 3 files changed, 1021 insertions(+), 109 deletions(-) diff --git a/js/src/experimental-prompt-adapters.ts b/js/src/experimental-prompt-adapters.ts index 2ed91e57e..5bd31ca5d 100644 --- a/js/src/experimental-prompt-adapters.ts +++ b/js/src/experimental-prompt-adapters.ts @@ -1,7 +1,12 @@ +import { BaseAttachment, ReadonlyAttachment } from "./logger"; +import type { BraintrustState } from "./logger"; +import type { AttachmentReferenceType as AttachmentReference } from "./generated_types"; import type { + InlineAttachmentReference, PromptDependencies, PromptJsonSchema, PromptMessage, + PromptMessageContentPart, } from "./experimental-prompt-api"; type PromptAdapterInput = { @@ -15,9 +20,24 @@ type PromptAdapterInput = { dependencies: PromptDependencies; }; +type OpenAIContentPart = + | { type: "text"; text: string } + | { + type: "image_url"; + image_url: { url: string; detail?: "auto" | "low" | "high" }; + } + | { + type: "file"; + file: { file_data?: string; file_id?: string; filename?: string }; + }; + +type OpenAIChatMessage = Omit & { + content: string | OpenAIContentPart[]; +}; + type OpenAIChatPromptArgs = { model?: string; - messages: PromptMessage[]; + messages: OpenAIChatMessage[]; response_format?: { type: "json_schema"; json_schema: { @@ -33,9 +53,18 @@ type OpenAIChatPromptArgs = { }; }; +type AISDKContentPart = + | { type: "text"; text: string } + | { type: "image"; image: string; mediaType?: string } + | { type: "file"; data: string; mediaType: string; filename?: string }; + +type AISDKMessage = Omit & { + content: string | AISDKContentPart[]; +}; + type AISDKGenerateObjectPromptArgs = { model?: string; - messages: PromptMessage[]; + messages: AISDKMessage[]; schema?: PromptJsonSchema; experimental_telemetry: { metadata: { @@ -44,8 +73,19 @@ type AISDKGenerateObjectPromptArgs = { }; }; -type OpenAIChatAdapterOptions = Record; -type AISDKGenerateObjectAdapterOptions = Record; +type AdapterOptions = { + state?: BraintrustState; +}; + +type OpenAIChatAdapterOptions = AdapterOptions; +type AISDKGenerateObjectAdapterOptions = AdapterOptions; + +type ResolvedPromptFile = { + data: string; + contentType?: string; + filename?: string; + detail?: "auto" | "low" | "high"; +}; function schemaName(slug: string): string { const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); @@ -54,13 +94,16 @@ function schemaName(slug: string): string { function openAIChatAdapter( options: OpenAIChatAdapterOptions = {}, -): (builtPrompt: PromptAdapterInput) => OpenAIChatPromptArgs { - void options; - return (builtPrompt) => { +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => { const outputSchema = builtPrompt.outputSchema?.toJSONSchema(); return { model: builtPrompt.model, - messages: builtPrompt.messages, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderOpenAIMessage(message, options), + ), + ), ...(outputSchema ? { response_format: { @@ -82,13 +125,64 @@ function openAIChatAdapter( }; } +async function renderOpenAIMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as OpenAIChatMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderOpenAIContentPart(part, options)), + ), + }; +} + +async function renderOpenAIContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image_url", + image_url: { + url: resolved.data, + ...(resolved.detail ? { detail: resolved.detail } : undefined), + }, + }; + } + + return { + type: "file", + file: isLikelyFileId(resolved.data) + ? { + file_id: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + } + : { + file_data: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }, + }; +} + function aiSDKGenerateObjectAdapter( options: AISDKGenerateObjectAdapterOptions = {}, -): (builtPrompt: PromptAdapterInput) => AISDKGenerateObjectPromptArgs { - void options; - return (builtPrompt) => ({ +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => ({ model: builtPrompt.model, - messages: builtPrompt.messages, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderAISDKMessage(message, options), + ), + ), schema: builtPrompt.outputSchema?.toJSONSchema(), experimental_telemetry: { metadata: { @@ -98,6 +192,236 @@ function aiSDKGenerateObjectAdapter( }); } +async function renderAISDKMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as AISDKMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderAISDKContentPart(part, options)), + ), + }; +} + +async function renderAISDKContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image", + image: resolved.data, + ...(resolved.contentType + ? { mediaType: resolved.contentType } + : undefined), + }; + } + + return { + type: "file", + data: resolved.data, + mediaType: resolved.contentType ?? "application/octet-stream", + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }; +} + +async function resolvePromptFile( + part: Extract, + options: AdapterOptions, +): Promise { + const value = part.file.value; + const optionContentType = part.file.contentType; + const optionFilename = part.file.filename; + + if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { + const reference = value.reference; + const data = + value instanceof ReadonlyAttachment + ? await value.asBase64Url() + : await blobToDataUrl(await value.data(), reference.content_type); + return { + data, + contentType: optionContentType ?? reference.content_type, + filename: optionFilename ?? reference.filename, + detail: part.file.detail, + }; + } + + if (isAttachmentReference(value)) { + const data = await new ReadonlyAttachment( + value, + options.state, + ).asBase64Url(); + return { + data, + contentType: optionContentType ?? value.content_type, + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isInlineAttachmentReference(value)) { + const data = value.data ?? value.src; + return { + data, + contentType: + optionContentType ?? value.content_type ?? contentTypeFromString(data), + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isBlob(value)) { + const contentType = optionContentType ?? (value.type || undefined); + return { + data: await blobToDataUrl(value, contentType), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) { + const contentType = optionContentType ?? "application/octet-stream"; + return { + data: await blobToDataUrl( + new Blob([value as BlobPart], { type: contentType }), + contentType, + ), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (typeof value === "string") { + return { + data: value, + contentType: optionContentType ?? contentTypeFromString(value), + filename: optionFilename, + detail: part.file.detail, + }; + } + + throw new Error("prompt.file value must be an attachment-compatible value"); +} + +function isImageFile(file: ResolvedPromptFile): boolean { + if (file.contentType?.startsWith("image/")) { + return true; + } + if (file.contentType && !file.contentType.startsWith("image/")) { + return false; + } + return isHttpUrl(file.data); +} + +function isLikelyFileId(value: string): boolean { + return !isHttpUrl(value) && !value.startsWith("data:"); +} + +function contentTypeFromString(value: string): string | undefined { + const dataUrlContentType = value.match(/^data:([^;,]+)[;,]/)?.[1]; + if (dataUrlContentType) { + return dataUrlContentType; + } + + const extension = value.split(/[?#]/, 1)[0]?.split(".").at(-1)?.toLowerCase(); + switch (extension) { + case "png": + return "image/png"; + case "jpg": + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "pdf": + return "application/pdf"; + case "txt": + return "text/plain"; + case "json": + return "application/json"; + default: + return undefined; + } +} + +function isHttpUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== "undefined" && value instanceof Blob; +} + +function isAttachmentReference(value: unknown): value is AttachmentReference { + return ( + isRecord(value) && + ((value.type === "braintrust_attachment" && + typeof value.key === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string") || + (value.type === "external_attachment" && + typeof value.url === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string")) + ); +} + +function isInlineAttachmentReference( + value: unknown, +): value is InlineAttachmentReference { + return ( + isRecord(value) && + value.type === "inline_attachment" && + typeof value.src === "string" && + (value.content_type === undefined || + typeof value.content_type === "string") && + (value.filename === undefined || typeof value.filename === "string") && + (value.data === undefined || typeof value.data === "string") + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function blobToDataUrl( + blob: Blob, + contentType?: string, +): Promise { + const buffer = await blob.arrayBuffer(); + const base64 = + typeof Buffer !== "undefined" + ? Buffer.from(buffer).toString("base64") + : bytesToBase64(new Uint8Array(buffer)); + return `data:${contentType ?? (blob.type || "application/octet-stream")};base64,${base64}`; +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + return btoa(binary); +} + export const adapters = { openAIChat: openAIChatAdapter, aiSDKGenerateObject: aiSDKGenerateObjectAdapter, diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts index 6cb197a96..2e4690dc6 100644 --- a/js/src/experimental-prompt-api.test.ts +++ b/js/src/experimental-prompt-api.test.ts @@ -1,8 +1,9 @@ import { describe, expect, test } from "vitest"; import { prompt, promptDefinitionToMustache } from "./experimental-prompt-api"; +import { Attachment, ReadonlyAttachment } from "./logger"; describe("experimental prompt API", () => { - test("builds message prompts and translates to OpenAI chat args", () => { + test("builds message prompts and translates to OpenAI chat args", async () => { const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", @@ -37,38 +38,40 @@ describe("experimental prompt API", () => { expect(built.definition.outputSchema?.parse(output, "output")).toEqual( output, ); - expect(built.to(prompt.adapters.openAIChat())).toMatchObject({ - model: "gpt-4o", - messages: [ - { role: "system", content: "You write concise support replies." }, - { role: "user", content: "Ticket: I cannot find eval history." }, - ], - response_format: { - type: "json_schema", - json_schema: { - name: "support-reply_output", - strict: true, - schema: { - type: "object", - required: ["subject", "body", "urgency"], + await expect(built.to(prompt.adapters.openAIChat())).resolves.toMatchObject( + { + model: "gpt-4o", + messages: [ + { role: "system", content: "You write concise support replies." }, + { role: "user", content: "Ticket: I cannot find eval history." }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "support-reply_output", + strict: true, + schema: { + type: "object", + required: ["subject", "body", "urgency"], + }, }, }, - }, - span_info: { - metadata: { - prompt: { - root: { slug: "support-reply" }, - prompts: [ - { - slug: "support-reply", - role: "root", - input: { ticket: "I cannot find eval history." }, - }, - ], + span_info: { + metadata: { + prompt: { + root: { slug: "support-reply" }, + prompts: [ + { + slug: "support-reply", + role: "root", + input: { ticket: "I cannot find eval history." }, + }, + ], + }, }, }, }, - }); + ); if (false) { // @ts-expect-error message prompts do not expose string content @@ -76,7 +79,7 @@ describe("experimental prompt API", () => { } }); - test("builds string prompts and coerces adapters to a user message", () => { + test("builds string prompts and coerces adapters to a user message", async () => { const policyText = prompt.define({ slug: "policy-text", model: "gpt-4o-mini", @@ -92,17 +95,19 @@ describe("experimental prompt API", () => { expect(built.kind).toBe("string"); expect(built.content).toBe("Policy: Prefer short answers."); expect("messages" in built).toBe(false); - expect(built.to(prompt.adapters.openAIChat())).toMatchObject({ - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Policy: Prefer short answers." }], - span_info: { - metadata: { - prompt: { - root: { slug: "policy-text" }, + await expect(built.to(prompt.adapters.openAIChat())).resolves.toMatchObject( + { + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Policy: Prefer short answers." }], + span_info: { + metadata: { + prompt: { + root: { slug: "policy-text" }, + }, }, }, }, - }); + ); expect( built.to((snapshot) => ({ kind: snapshot.kind, @@ -397,7 +402,168 @@ describe("experimental prompt API", () => { ); }); - test("extends adapter args with a typed deep merge", () => { + test("converts prompt.file parts for OpenAI and AI SDK adapters", async () => { + const imageDataUrl = "data:image/png;base64,aW1hZ2U="; + const pdfDataUrl = "data:application/pdf;base64,cGRm"; + const describeFiles = prompt.define({ + slug: "describe-files", + model: "gpt-4o", + input: (s) => + s.object({ + image: s.attachment(), + document: s.attachment(), + gallery: s.array(s.attachment()), + }), + render: ({ variables, values }) => [ + prompt.user([ + prompt.text`Describe these files.`, + prompt.file(variables.image), + prompt.file(variables.document, { filename: "brief.pdf" }), + ...values.gallery.map((item) => prompt.file(item)), + ]), + ], + }); + + const built = describeFiles.build({ + image: imageDataUrl, + document: pdfDataUrl, + gallery: [ + "https://example.com/first.png", + "https://example.com/second.jpg", + ], + }); + const openAIArgs = await built.to(prompt.adapters.openAIChat()); + const aiSDKArgs = await built.to(prompt.adapters.aiSDKGenerateObject()); + + expect(openAIArgs.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these files." }, + { type: "image_url", image_url: { url: imageDataUrl } }, + { + type: "file", + file: { file_data: pdfDataUrl, filename: "brief.pdf" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/first.png" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/second.jpg" }, + }, + ]); + expect(aiSDKArgs.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these files." }, + { type: "image", image: imageDataUrl, mediaType: "image/png" }, + { + type: "file", + data: pdfDataUrl, + mediaType: "application/pdf", + filename: "brief.pdf", + }, + { + type: "image", + image: "https://example.com/first.png", + mediaType: "image/png", + }, + { + type: "image", + image: "https://example.com/second.jpg", + mediaType: "image/jpeg", + }, + ]); + }); + + test("resolves Attachment and ReadonlyAttachment values without leaking bytes", async () => { + const attachment = new Attachment({ + data: new Blob(["hello"], { type: "text/plain" }), + filename: "hello.txt", + contentType: "text/plain", + }); + const readonly = new ReadonlyAttachment({ + type: "external_attachment", + url: "https://example.com/doc.pdf", + filename: "doc.pdf", + content_type: "application/pdf", + }); + readonly.asBase64Url = async () => "data:application/pdf;base64,cGRm"; + const describeFiles = prompt.define({ + slug: "describe-uploaded-files", + model: "gpt-4o", + input: (s) => + s.object({ + attachment: s.attachment(), + readonly: s.attachment(), + }), + render: ({ variables }) => [ + prompt.user([ + prompt.text`Describe these uploads.`, + prompt.file(variables.attachment), + prompt.file(variables.readonly), + ]), + ], + }); + + const built = describeFiles.build({ attachment, readonly }); + const args = await built.to(prompt.adapters.openAIChat()); + + expect(args.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these uploads." }, + { + type: "file", + file: { + file_data: "data:text/plain;base64,aGVsbG8=", + filename: "hello.txt", + }, + }, + { + type: "file", + file: { + file_data: "data:application/pdf;base64,cGRm", + filename: "doc.pdf", + }, + }, + ]); + expect(JSON.stringify(built.dependencies.prompts[0]?.input)).not.toContain( + "aGVsbG8=", + ); + expect(built.dependencies.prompts[0]?.input).toMatchObject({ + attachment: { + type: "attachment", + reference: { + type: "braintrust_attachment", + filename: "hello.txt", + content_type: "text/plain", + }, + }, + readonly: { + type: "attachment", + reference: { + type: "external_attachment", + filename: "doc.pdf", + content_type: "application/pdf", + }, + }, + }); + }); + + test("rejects rich media content outside user messages", () => { + const invalid = prompt.define({ + slug: "invalid-media-role", + input: (s) => s.object({}), + render: () => [ + { + role: "system" as const, + content: [prompt.file("https://example.com/image.png")], + }, + ], + }); + + expect(() => invalid.build({})).toThrow( + "render[0] must be a prompt message", + ); + }); + + test("extends adapter args with a typed deep merge", async () => { const classify = prompt.define({ slug: "classify", model: "gpt-4o-mini", @@ -413,7 +579,7 @@ describe("experimental prompt API", () => { }); const built = classify.build({ text: "It crashes" }); - const args = built.to(prompt.adapters.openAIChat()); + const args = await built.to(prompt.adapters.openAIChat()); const extended = args.extend({ temperature: 0.2, span_info: { @@ -471,23 +637,27 @@ describe("experimental prompt API", () => { // @ts-expect-error adapters must return objects void built.to(() => "nope"); - const typedArgs = built.to(prompt.adapters.openAIChat()).extend({ - temperature: 0.2, - span_info: { - metadata: { - caller: "support-workflow", + void (async () => { + const typedArgs = (await built.to(prompt.adapters.openAIChat())).extend( + { + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, + }, }, - }, + ); + const temperature: number = typedArgs.temperature; + const caller: string = typedArgs.span_info.metadata.caller; + const slug: string = typedArgs.span_info.metadata.prompt.root.slug; + void temperature; + void caller; + void slug; + + // @ts-expect-error extend only accepts objects + void typedArgs.extend("nope"); }); - const temperature: number = typedArgs.temperature; - const caller: string = typedArgs.span_info.metadata.caller; - const slug: string = typedArgs.span_info.metadata.prompt.root.slug; - void temperature; - void caller; - void slug; - - // @ts-expect-error extend only accepts objects - void typedArgs.extend("nope"); } }); @@ -993,6 +1163,7 @@ describe("experimental prompt API", () => { expect("builtStringPrompt" in s).toBe(true); expect("messagesPromptDefinition" in s).toBe(true); expect("stringPromptDefinition" in s).toBe(true); + expect("attachment" in s).toBe(true); expect("prompt" in s).toBe(false); expect("builtPrompt" in s).toBe(false); expect("promptDefinition" in s).toBe(false); @@ -1012,6 +1183,8 @@ describe("experimental prompt API", () => { void s.builtPrompt; // @ts-expect-error prompt definitions are no longer accepted as schema inputs void s.promptDefinition; + // @ts-expect-error rich content is only supported by prompt.user + void prompt.system([prompt.file("https://example.com/image.png")]); } return s.object({}); diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts index 575e25c3b..4b7ff4df8 100644 --- a/js/src/experimental-prompt-api.ts +++ b/js/src/experimental-prompt-api.ts @@ -1,4 +1,10 @@ import { adapters } from "./experimental-prompt-adapters"; +import { BaseAttachment, ReadonlyAttachment } from "./logger"; +import type { + AttachmentReferenceType as AttachmentReference, + ChatCompletionContentPartType, + ChatCompletionMessageParamType, +} from "./generated_types"; import type { PromptDefinition as MustachePromptDefinition } from "./prompt-schemas"; type JsonPrimitive = string | number | boolean | null; @@ -32,6 +38,9 @@ type PromptSchemaTemplateInfo = type: "promptDefinition"; definition: AnyPromptDefinition; kind: PromptKind; + } + | { + type: "attachment"; }; class PromptSchema< @@ -157,6 +166,7 @@ const promptDefinitionMarker = Symbol( "braintrust.experimental_prompt.definition", ); const promptTextMarker = Symbol("braintrust.experimental_prompt.text"); +const promptFileMarker = Symbol("braintrust.experimental_prompt.file"); const promptDependencyMarker = Symbol( "braintrust.experimental_prompt.dependencies", ); @@ -168,9 +178,60 @@ const templateValueStateMarker = Symbol( ); type PromptRole = "system" | "user" | "assistant"; + +export type InlineAttachmentReference = { + type: "inline_attachment"; + src: string; + content_type?: string; + filename?: string; + data?: string; +}; + +export type PromptAttachment = + | string + | Blob + | ArrayBuffer + | ArrayBufferView + | BaseAttachment + | ReadonlyAttachment + | AttachmentReference + | InlineAttachmentReference; + +type PromptFileOptions = { + filename?: string; + contentType?: string; + detail?: "auto" | "low" | "high"; +}; + +export type PromptTextContentPart = { + type: "text"; + text: string; +}; + +export type PromptFileContentPart = { + readonly [promptFileMarker]: true; + type: "file"; + file: { + value: unknown; + filename?: string; + contentType?: string; + detail?: "auto" | "low" | "high"; + }; +}; + +export type PromptMessageContentPart = + | PromptTextContentPart + | PromptFileContentPart; + +type PromptUserContentPartInput = + | string + | PromptText + | PromptTextContentPart + | PromptFileContentPart; + export type PromptMessage = { role: PromptRole; - content: string; + content: string | PromptMessageContentPart[]; }; type PromptMessageWithDependencies = PromptMessage & { @@ -376,10 +437,15 @@ class PromptDefinition< }; if (isPromptText(rendered)) { - const dependencies = createPromptDependencies(root, parsedInput, [ - ...collectBuiltPromptDependencies(parsedInput, this.slug), - ...collectDependencyEntries(rendered.dependencies, this.slug), - ]); + const dependencies = createPromptDependencies( + root, + parsedInput, + [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectDependencyEntries(rendered.dependencies, this.slug), + ], + this.inputSchema, + ); return new BuiltStringPrompt< InferSchema, @@ -419,10 +485,15 @@ class PromptDefinition< const messages = rendered.map((message, index) => assertPromptMessage(message, `render[${index}]`), ); - const dependencies = createPromptDependencies(root, parsedInput, [ - ...collectBuiltPromptDependencies(parsedInput, this.slug), - ...collectMessageDependencies(messages, this.slug), - ]); + const dependencies = createPromptDependencies( + root, + parsedInput, + [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectMessageDependencies(messages, this.slug), + ], + this.inputSchema, + ); return new BuiltMessagesPrompt< InferSchema, @@ -715,6 +786,7 @@ type DeepMergeValue = TBase extends readonly unknown[] type PromptExtension = Record; type PromptAdapterResult = PromptExtension; +type MaybePromise = T | Promise; type Extendable = T & { extend( @@ -724,7 +796,11 @@ type Extendable = T & { type PromptAdapter = ( builtPrompt: PromptAdapterInput, -) => TResult; +) => MaybePromise; + +type PromptAdapterToResult = + | Extendable + | Promise>; class BuiltMessagesPrompt implements Iterable { readonly [builtPromptMarker] = true; @@ -755,18 +831,19 @@ class BuiltMessagesPrompt implements Iterable { to( adapter: PromptAdapter, - ): Extendable { - return makeExtendableAdapterResult( - adapter({ - kind: "messages", - model: this.definition.model, - inputSchema: this.definition.inputSchema, - outputSchema: this.definition.outputSchema, - input: this.input, - messages: this.messages, - dependencies: this.dependencies, - }), - ); + ): PromptAdapterToResult { + const result = adapter({ + kind: "messages", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + messages: this.messages, + dependencies: this.dependencies, + }); + return isPromiseLike(result) + ? result.then((resolved) => makeExtendableAdapterResult(resolved)) + : makeExtendableAdapterResult(result); } } @@ -791,22 +868,27 @@ class BuiltStringPrompt { to( adapter: PromptAdapter, - ): Extendable { - return makeExtendableAdapterResult( - adapter({ - kind: "string", - model: this.definition.model, - inputSchema: this.definition.inputSchema, - outputSchema: this.definition.outputSchema, - input: this.input, - content: this.content, - messages: [{ role: "user", content: this.content }], - dependencies: this.dependencies, - }), - ); + ): PromptAdapterToResult { + const result = adapter({ + kind: "string", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + content: this.content, + messages: [{ role: "user", content: this.content }], + dependencies: this.dependencies, + }); + return isPromiseLike(result) + ? result.then((resolved) => makeExtendableAdapterResult(resolved)) + : makeExtendableAdapterResult(result); } } +function isPromiseLike(value: MaybePromise): value is Promise { + return isRecord(value) && typeof value.then === "function"; +} + function makeExtendableAdapterResult( result: TResult, ): Extendable { @@ -866,19 +948,93 @@ function definePrompt< return new PromptDefinition(opts); } -function messageTag(role: PromptRole) { - return ( - strings: TemplateStringsArray, +type PromptMessageTag = ( + strings: TemplateStringsArray, + ...values: readonly unknown[] +) => PromptMessage; + +type PromptUserMessageTag = PromptMessageTag & { + (content: readonly PromptUserContentPartInput[]): PromptMessage; +}; + +function messageTag(role: "system" | "assistant"): PromptMessageTag; +function messageTag(role: "user"): PromptUserMessageTag; +function messageTag(role: PromptRole): PromptMessageTag | PromptUserMessageTag { + return (( + stringsOrContent: + | TemplateStringsArray + | readonly PromptUserContentPartInput[], ...values: readonly unknown[] ): PromptMessage => { - const rendered = renderTaggedTemplate(strings, values); + if (!isTemplateStringsArray(stringsOrContent)) { + if (role !== "user") { + throw new Error( + "rich prompt content is only supported for user messages", + ); + } + return userMessageFromContentParts(stringsOrContent); + } + + const rendered = renderTaggedTemplate(stringsOrContent, values); return attachDependenciesToMessage( { role, content: rendered.content }, rendered.dependencies, ); + }) as PromptMessageTag | PromptUserMessageTag; +} + +function userMessageFromContentParts( + parts: readonly PromptUserContentPartInput[], +): PromptMessage { + const dependencies: PromptDependencyEntry[] = []; + const content = parts.map((part, index): PromptMessageContentPart => { + if (typeof part === "string") { + return { type: "text", text: part }; + } + if (isPromptText(part)) { + dependencies.push(...part.dependencies); + return { type: "text", text: part.content }; + } + if (isPromptFileContentPart(part)) { + return part; + } + if ( + isRecord(part) && + part.type === "text" && + typeof part.text === "string" + ) { + return { type: "text", text: part.text }; + } + + throw new Error( + `user content part ${index} must be prompt.text or prompt.file`, + ); + }); + return attachDependenciesToMessage({ role: "user", content }, dependencies); +} + +function filePart( + value: unknown, + options: PromptFileOptions = {}, +): PromptFileContentPart { + return { + [promptFileMarker]: true, + type: "file", + file: { + value, + filename: options.filename, + contentType: options.contentType, + detail: options.detail, + }, }; } +function isTemplateStringsArray(value: unknown): value is TemplateStringsArray { + return ( + Array.isArray(value) && Array.isArray((value as { raw?: unknown }).raw) + ); +} + function textTag( strings: TemplateStringsArray, ...values: readonly unknown[] @@ -987,13 +1143,38 @@ function assertPromptMessage(value: unknown, path: string): PromptMessage { (value.role !== "system" && value.role !== "user" && value.role !== "assistant") || - typeof value.content !== "string" + !isPromptMessageContent(value.role, value.content) ) { throw new Error(`${path} must be a prompt message`); } return value as PromptMessage; } +function isPromptMessageContent(role: unknown, content: unknown): boolean { + if (typeof content === "string") { + return true; + } + if (!Array.isArray(content)) { + return false; + } + if (role !== "user") { + return false; + } + return content.every(isPromptMessageContentPart); +} + +function isPromptMessageContentPart( + value: unknown, +): value is PromptMessageContentPart { + if (!isRecord(value) || typeof value.type !== "string") { + return false; + } + if (value.type === "text") { + return typeof value.text === "string"; + } + return isPromptFileContentPart(value); +} + function createRootTemplateScope( inputSchema: InputSchema, ): PromptTemplateScope { @@ -1074,6 +1255,18 @@ function createPromptVariables( : nestedPromptBuilder(templateInfo.definition, templateInfo.kind, path); } + if (templateInfo?.type === "attachment") { + return mode === "runtime" + ? runtimeValue + : createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }); + } + return createPromptVariableValue({ path, mode, @@ -1387,6 +1580,7 @@ function createPromptDependencies( root: PromptDependencies["root"], input: unknown, entries: PromptDependencyEntry[], + inputSchema?: InputSchema, ): PromptDependencies { return { root, @@ -1394,7 +1588,7 @@ function createPromptDependencies( { ...root, role: "root", - input: sanitizeDependencyInput(input), + input: sanitizeDependencyInput(input, inputSchema), }, ...dedupeDependencyEntries(entries), ], @@ -1428,6 +1622,9 @@ function collectBuiltPromptDependencies( value: unknown, parent: string, ): PromptDependencyEntry[] { + if (isPromptAttachmentValue(value) || isPromptFileContentPart(value)) { + return []; + } if (isBuiltPrompt(value)) { return collectDependencyEntries(value.dependencies.prompts, parent); } @@ -1458,7 +1655,27 @@ function dedupeDependencyEntries( }); } -function sanitizeDependencyInput(value: unknown): unknown { +function sanitizeDependencyInput( + value: unknown, + schema?: InputSchema, +): unknown { + const templateInfo = schema?.templateInfo; + if (templateInfo?.type === "attachment") { + return summarizeAttachmentInput(value); + } + if (templateInfo?.type === "object" && isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + sanitizeDependencyInput(item, templateInfo.shape[key] as InputSchema), + ]), + ); + } + if (templateInfo?.type === "array" && Array.isArray(value)) { + return value.map((item) => + sanitizeDependencyInput(item, templateInfo.item as InputSchema), + ); + } if (isBuiltPrompt(value)) { return { type: @@ -1475,6 +1692,20 @@ function sanitizeDependencyInput(value: unknown): unknown { version: value.version, }; } + if (isPromptFileContentPart(value)) { + return { + type: "prompt_file", + file: summarizeAttachmentInput(value.file.value), + filename: value.file.filename, + content_type: value.file.contentType, + }; + } + if (isPromptAttachmentValue(value)) { + return summarizeAttachmentInput(value); + } + if (typeof value === "string" && value.startsWith("data:")) { + return summarizeStringAttachment(value); + } if (Array.isArray(value)) { return value.map((item) => sanitizeDependencyInput(item)); } @@ -1489,6 +1720,57 @@ function sanitizeDependencyInput(value: unknown): unknown { return value; } +function summarizeAttachmentInput(value: unknown): unknown { + if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { + return { + type: "attachment", + reference: value.reference, + }; + } + if (isAttachmentReference(value)) { + return value; + } + if (isInlineAttachmentReference(value)) { + return { + type: "inline_attachment", + content_type: value.content_type, + filename: value.filename, + src: summarizeStringAttachment(value.src), + }; + } + if (isBlob(value)) { + return { + type: "blob", + content_type: value.type || undefined, + byte_length: value.size, + }; + } + if (value instanceof ArrayBuffer) { + return { type: "array_buffer", byte_length: value.byteLength }; + } + if (ArrayBuffer.isView(value)) { + return { + type: "binary", + byte_length: value.byteLength, + }; + } + if (typeof value === "string") { + return summarizeStringAttachment(value); + } + return { type: "attachment", value_type: typeof value }; +} + +function summarizeStringAttachment(value: string): unknown { + if (value.startsWith("data:")) { + return { + type: "data_url", + content_type: dataUrlContentType(value), + byte_length: value.length, + }; + } + return value; +} + function isBuiltPrompt(value: unknown): value is AnyBuiltPrompt { return ( typeof value === "object" && value !== null && builtPromptMarker in value @@ -1521,6 +1803,63 @@ function isPromptText(value: unknown): value is PromptText { ); } +function isPromptFileContentPart( + value: unknown, +): value is PromptFileContentPart { + return ( + typeof value === "object" && value !== null && promptFileMarker in value + ); +} + +function isPromptAttachmentValue(value: unknown): value is PromptAttachment { + return ( + typeof value === "string" || + isBlob(value) || + value instanceof ArrayBuffer || + ArrayBuffer.isView(value) || + value instanceof BaseAttachment || + value instanceof ReadonlyAttachment || + isAttachmentReference(value) || + isInlineAttachmentReference(value) + ); +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== "undefined" && value instanceof Blob; +} + +function isAttachmentReference(value: unknown): value is AttachmentReference { + return ( + isRecord(value) && + ((value.type === "braintrust_attachment" && + typeof value.key === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string") || + (value.type === "external_attachment" && + typeof value.url === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string")) + ); +} + +function isInlineAttachmentReference( + value: unknown, +): value is InlineAttachmentReference { + return ( + isRecord(value) && + value.type === "inline_attachment" && + typeof value.src === "string" && + (value.content_type === undefined || + typeof value.content_type === "string") && + (value.filename === undefined || typeof value.filename === "string") && + (value.data === undefined || typeof value.data === "string") + ); +} + +function dataUrlContentType(value: string): string | undefined { + return value.match(/^data:([^;,]+)[;,]/)?.[1]; +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -1748,6 +2087,25 @@ function unknownSchema(): PromptSchema< ); } +function attachmentSchema(): PromptSchema< + PromptAttachment, + PromptAttachment, + unknown, + "input" +> { + return new PromptSchema( + (value, path) => { + if (!isPromptAttachmentValue(value)) { + throw new Error(`${path} must be an attachment`); + } + return value; + }, + () => ({ "x-bt-type": "attachment" }), + false, + { type: "attachment" }, + ); +} + function builtMessagesPromptSchema(): PromptSchema< BuiltMessagesPrompt, BuiltMessagesPrompt, @@ -1889,7 +2247,7 @@ export function promptDefinitionToMustache( if (data.kind === "messages") { return { model: data.model, - messages: data.messages, + messages: data.messages.map(promptMessageToMustacheMessage), }; } @@ -1899,6 +2257,61 @@ export function promptDefinitionToMustache( }; } +function promptMessageToMustacheMessage( + message: PromptMessage, +): ChatCompletionMessageParamType { + if (typeof message.content === "string") { + if (message.role === "system") { + return { role: "system", content: message.content }; + } + if (message.role === "assistant") { + return { role: "assistant", content: message.content }; + } + return { role: "user", content: message.content }; + } + if (message.role !== "user") { + throw new Error("Only user messages can contain prompt.file parts"); + } + return { + role: "user", + content: message.content.map(promptContentPartToMustachePart), + }; +} + +function promptContentPartToMustachePart( + part: PromptMessageContentPart, +): ChatCompletionContentPartType { + if (part.type === "text") { + return part; + } + + const value = stringifyTemplateValue(part.file.value).content; + const contentType = + part.file.contentType ?? + (typeof value === "string" ? dataUrlContentType(value) : undefined); + if (isImageContentType(contentType)) { + return { + type: "image_url" as const, + image_url: { + url: value, + ...(part.file.detail ? { detail: part.file.detail } : undefined), + }, + }; + } + + return { + type: "file" as const, + file: { + file_data: value, + ...(part.file.filename ? { filename: part.file.filename } : undefined), + }, + }; +} + +function isImageContentType(contentType: string | undefined): boolean { + return contentType?.startsWith("image/") ?? false; +} + const inputSchemaHelpers = { string: stringSchema, number: numberSchema, @@ -1907,6 +2320,7 @@ const inputSchemaHelpers = { array: arraySchema, object: objectSchema, unknown: unknownSchema, + attachment: attachmentSchema, builtMessagesPrompt: builtMessagesPromptSchema, builtStringPrompt: builtStringPromptSchema, messagesPromptDefinition: messagesPromptDefinitionSchema, @@ -1931,6 +2345,7 @@ export const prompt = { user: messageTag("user"), assistant: messageTag("assistant"), text: textTag, + file: filePart, isBuiltPrompt, isPromptDefinition, adapters, From 36cb9a1a938961c7851dce4d21c720b0e74a2f20 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Fri, 3 Jul 2026 16:57:21 +0200 Subject: [PATCH 5/6] rename and no values in template --- js/src/experimental-prompt-api.test.ts | 258 ++++++++++--------------- js/src/experimental-prompt-api.ts | 110 ++++------- 2 files changed, 144 insertions(+), 224 deletions(-) diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts index 2e4690dc6..8f5795465 100644 --- a/js/src/experimental-prompt-api.test.ts +++ b/js/src/experimental-prompt-api.test.ts @@ -7,17 +7,17 @@ describe("experimental prompt API", () => { const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ ticket: s.string(), }), - output: (s) => + outputSchema: (s) => s.object({ subject: s.string(), body: s.string(), urgency: s.enum(["low", "medium", "high"]), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`You write concise support replies.`, prompt.user`Ticket: ${variables.ticket}`, ], @@ -83,11 +83,11 @@ describe("experimental prompt API", () => { const policyText = prompt.define({ slug: "policy-text", model: "gpt-4o-mini", - input: (s) => + inputSchema: (s) => s.object({ policy: s.string(), }), - render: ({ variables }) => prompt.text`Policy: ${variables.policy}`, + template: ({ variables }) => prompt.text`Policy: ${variables.policy}`, }); const built = policyText.build({ policy: "Prefer short answers." }); @@ -132,18 +132,18 @@ describe("experimental prompt API", () => { const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ customer: s.object({ name: s.string(), }), ticket: s.string(), }), - output: (s) => + outputSchema: (s) => s.object({ body: s.string(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`You write concise support replies.`, prompt.user`Customer: ${variables.customer.name}\nTicket: ${variables.ticket}`, ], @@ -187,11 +187,11 @@ describe("experimental prompt API", () => { const summarize = prompt.define({ slug: "summarize", model: "gpt-4o-mini", - input: (s) => + inputSchema: (s) => s.object({ text: s.string(), }), - render: ({ variables }) => prompt.text`Summarize: ${variables.text}`, + template: ({ variables }) => prompt.text`Summarize: ${variables.text}`, }); const data = summarize.toPromptData(); @@ -212,25 +212,25 @@ describe("experimental prompt API", () => { const brandVoice = prompt.define({ slug: "brand-voice", version: "v3", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), tone: s.string(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, ], }); const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), ticket: s.string(), voice: s.messagesPromptDefinition(brandVoice), }), - render: ({ variables }) => [ + template: ({ variables }) => [ ...variables.voice, prompt.user`Draft a reply for: ${variables.ticket}`, ], @@ -261,24 +261,24 @@ describe("experimental prompt API", () => { const policyText = prompt.define({ slug: "policy-text", version: "v2", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), text: s.string(), }), - render: ({ variables }) => + template: ({ variables }) => prompt.text`${variables.company}: ${variables.text}`, }); const supportReply = prompt.define({ slug: "support-reply", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), ticket: s.string(), policy: s.stringPromptDefinition(policyText), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`Follow this policy: ${variables.policy}`, prompt.user`Draft a reply for: ${variables.ticket}`, ], @@ -308,8 +308,8 @@ describe("experimental prompt API", () => { test("promptDefinitionToMustache requires a model", () => { const modeless = prompt.define({ slug: "modeless", - input: (s) => s.object({ text: s.string() }), - render: ({ variables }) => [prompt.user`Say ${variables.text}`], + inputSchema: (s) => s.object({ text: s.string() }), + template: ({ variables }) => [prompt.user`Say ${variables.text}`], }); expect(() => promptDefinitionToMustache(modeless.toPromptData())).toThrow( @@ -321,7 +321,7 @@ describe("experimental prompt API", () => { const itemList = prompt.define({ slug: "item-list", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ items: s.array( s.object({ @@ -332,7 +332,7 @@ describe("experimental prompt API", () => { }), ), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.user`Items:\n${variables.items.list`- ${variables.items.list.foobar} by ${variables.items.list.author.name}\n`}`, ], }); @@ -371,35 +371,24 @@ describe("experimental prompt API", () => { }); }); - test("passes parsed runtime values separately from template variables", () => { - const itemSummary = prompt.define({ - slug: "item-summary", - input: (s) => - s.object({ - items: s.array( - s.object({ - foobar: s.string(), - }), - ), - }), - render: ({ values }) => [ - prompt.user`Second: ${values.items[1]?.foobar}\nAll: ${values.items.map((item) => item.foobar).join(", ")}`, - ], - }); - - expect( - itemSummary.build({ - items: [{ foobar: "first" }, { foobar: "second" }], - }).messages, - ).toEqual([ - { - role: "user", - content: "Second: second\nAll: first, second", - }, - ]); - expect(() => itemSummary.toPromptData()).toThrow( - "Runtime values are not available while exporting prompt data; use variables in prompt templates.", - ); + test("template contexts do not expose parsed runtime values", () => { + if (false) { + prompt.define({ + slug: "no-runtime-values", + inputSchema: (s) => + s.object({ + items: s.array( + s.object({ + foobar: s.string(), + }), + ), + }), + // @ts-expect-error template contexts only expose template variables + template: ({ values }) => [ + prompt.user`Second: ${values.items[1]?.foobar}`, + ], + }); + } }); test("converts prompt.file parts for OpenAI and AI SDK adapters", async () => { @@ -408,18 +397,18 @@ describe("experimental prompt API", () => { const describeFiles = prompt.define({ slug: "describe-files", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ image: s.attachment(), document: s.attachment(), - gallery: s.array(s.attachment()), + secondImage: s.attachment(), }), - render: ({ variables, values }) => [ + template: ({ variables }) => [ prompt.user([ prompt.text`Describe these files.`, prompt.file(variables.image), prompt.file(variables.document, { filename: "brief.pdf" }), - ...values.gallery.map((item) => prompt.file(item)), + prompt.file(variables.secondImage), ]), ], }); @@ -427,10 +416,7 @@ describe("experimental prompt API", () => { const built = describeFiles.build({ image: imageDataUrl, document: pdfDataUrl, - gallery: [ - "https://example.com/first.png", - "https://example.com/second.jpg", - ], + secondImage: "https://example.com/second.jpg", }); const openAIArgs = await built.to(prompt.adapters.openAIChat()); const aiSDKArgs = await built.to(prompt.adapters.aiSDKGenerateObject()); @@ -442,10 +428,6 @@ describe("experimental prompt API", () => { type: "file", file: { file_data: pdfDataUrl, filename: "brief.pdf" }, }, - { - type: "image_url", - image_url: { url: "https://example.com/first.png" }, - }, { type: "image_url", image_url: { url: "https://example.com/second.jpg" }, @@ -460,11 +442,6 @@ describe("experimental prompt API", () => { mediaType: "application/pdf", filename: "brief.pdf", }, - { - type: "image", - image: "https://example.com/first.png", - mediaType: "image/png", - }, { type: "image", image: "https://example.com/second.jpg", @@ -489,12 +466,12 @@ describe("experimental prompt API", () => { const describeFiles = prompt.define({ slug: "describe-uploaded-files", model: "gpt-4o", - input: (s) => + inputSchema: (s) => s.object({ attachment: s.attachment(), readonly: s.attachment(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.user([ prompt.text`Describe these uploads.`, prompt.file(variables.attachment), @@ -549,8 +526,8 @@ describe("experimental prompt API", () => { test("rejects rich media content outside user messages", () => { const invalid = prompt.define({ slug: "invalid-media-role", - input: (s) => s.object({}), - render: () => [ + inputSchema: (s) => s.object({}), + template: () => [ { role: "system" as const, content: [prompt.file("https://example.com/image.png")], @@ -559,7 +536,7 @@ describe("experimental prompt API", () => { }); expect(() => invalid.build({})).toThrow( - "render[0] must be a prompt message", + "template[0] must be a prompt message", ); }); @@ -567,15 +544,15 @@ describe("experimental prompt API", () => { const classify = prompt.define({ slug: "classify", model: "gpt-4o-mini", - input: (s) => + inputSchema: (s) => s.object({ text: s.string(), }), - output: (s) => + outputSchema: (s) => s.object({ label: s.enum(["bug", "question"]), }), - render: ({ variables }) => [prompt.user`Classify: ${variables.text}`], + template: ({ variables }) => [prompt.user`Classify: ${variables.text}`], }); const built = classify.build({ text: "It crashes" }); @@ -665,12 +642,12 @@ describe("experimental prompt API", () => { const brandVoice = prompt.define({ slug: "brand-voice", version: "v3", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), tone: s.string(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, ], }); @@ -678,13 +655,13 @@ describe("experimental prompt API", () => { const supportReply = prompt.define({ slug: "support-reply", version: "v8", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), ticket: s.string(), voice: s.messagesPromptDefinition(brandVoice), }), - render: ({ variables }) => [ + template: ({ variables }) => [ ...variables.voice, prompt.user`Draft a reply for: ${variables.ticket}`, ], @@ -773,24 +750,24 @@ describe("experimental prompt API", () => { const policyText = prompt.define({ slug: "policy-text", version: "v2", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), policy: s.string(), }), - render: ({ variables }) => + template: ({ variables }) => prompt.text`${variables.company}: ${variables.policy}`, }); const supportReply = prompt.define({ slug: "support-reply", - input: (s) => + inputSchema: (s) => s.object({ company: s.string(), ticket: s.string(), policy: s.stringPromptDefinition(policyText), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.system`Follow this policy: ${variables.policy}`, prompt.user`Draft a reply for: ${variables.ticket}`, ], @@ -851,31 +828,31 @@ describe("experimental prompt API", () => { test("requires matching built prompt kinds for dynamic schemas", () => { const messagePrompt = prompt.define({ slug: "message-prompt", - input: (s) => + inputSchema: (s) => s.object({ topic: s.string(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.user`Message about ${variables.topic}`, ], }); const stringPrompt = prompt.define({ slug: "string-prompt", - input: (s) => + inputSchema: (s) => s.object({ topic: s.string(), }), - render: ({ variables }) => prompt.text`String about ${variables.topic}`, + template: ({ variables }) => prompt.text`String about ${variables.topic}`, }); const consumeBoth = prompt.define({ slug: "consume-both", - input: (s) => + inputSchema: (s) => s.object({ messagePart: s.builtMessagesPrompt(), stringPart: s.builtStringPrompt(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ ...variables.messagePart, prompt.user`Fragment: ${variables.stringPart}`, ], @@ -930,29 +907,29 @@ describe("experimental prompt API", () => { test("preserves dependencies through spread and interpolation outside schema inputs", () => { const messagePrompt = prompt.define({ slug: "message-prompt", - input: (s) => + inputSchema: (s) => s.object({ topic: s.string(), }), - render: ({ variables }) => [ + template: ({ variables }) => [ prompt.user`Message about ${variables.topic}`, ], }); const stringPrompt = prompt.define({ slug: "string-prompt", - input: (s) => + inputSchema: (s) => s.object({ topic: s.string(), }), - render: ({ variables }) => prompt.text`String about ${variables.topic}`, + template: ({ variables }) => prompt.text`String about ${variables.topic}`, }); const messagePart = messagePrompt.build({ topic: "tracing" }); const stringPart = stringPrompt.build({ topic: "evals" }); const wrapper = prompt.define({ slug: "wrapper", - input: (s) => s.object({}), - render: () => [...messagePart, prompt.user`Fragment: ${stringPart}`], + inputSchema: (s) => s.object({}), + template: () => [...messagePart, prompt.user`Fragment: ${stringPart}`], }); const built = wrapper.build({}); @@ -964,24 +941,24 @@ describe("experimental prompt API", () => { ]); }); - test("validates input, output, and render shapes", () => { + test("validates build input, output, and template results", () => { const typedPrompt = prompt.define({ slug: "typed", - input: (s) => + inputSchema: (s) => s.object({ count: s.number(), }), - output: (s) => + outputSchema: (s) => s.object({ ok: s.boolean(), }), - render: ({ variables }) => [prompt.user`Count: ${variables.count}`], + template: ({ variables }) => [prompt.user`Count: ${variables.count}`], }); - const invalidRenderPrompt = prompt.define({ - slug: "invalid-render", - input: (s) => s.object({}), - // @ts-expect-error render must return a message array or prompt.text - render: () => prompt.user`Nope`, + const invalidTemplatePrompt = prompt.define({ + slug: "invalid-template", + inputSchema: (s) => s.object({}), + // @ts-expect-error template must return a message array or prompt.text + template: () => prompt.user`Nope`, }); expect(() => @@ -992,29 +969,29 @@ describe("experimental prompt API", () => { expect(() => built.definition.outputSchema?.parse({ ok: "yes" }, "output"), ).toThrow("output.ok must be a boolean"); - expect(() => invalidRenderPrompt.build({})).toThrow( - "render must return a message array or prompt.text", + expect(() => invalidTemplatePrompt.build({})).toThrow( + "template must return a message array or prompt.text", ); }); - test("passes scoped schema helpers to input and output callbacks", () => { + test("passes scoped schema helpers to inputSchema and outputSchema callbacks", () => { let inputHelperKeys: string[] = []; let outputHelperKeys: string[] = []; - const typedPrompt = prompt.define({ + prompt.define({ slug: "schema-helper-scopes", - input: (s) => { + inputSchema: (s) => { inputHelperKeys = Object.keys(s).sort(); return s.object({ topic: s.string(), }); }, - output: (s) => { + outputSchema: (s) => { outputHelperKeys = Object.keys(s).sort(); return s.object({ ok: s.boolean(), }); }, - render: ({ variables }) => [prompt.user`Topic: ${variables.topic}`], + template: ({ variables }) => [prompt.user`Topic: ${variables.topic}`], }); expect(inputHelperKeys).toContain("builtMessagesPrompt"); @@ -1026,80 +1003,49 @@ describe("experimental prompt API", () => { expect(outputHelperKeys).not.toContain("stringPromptDefinition"); if (false) { - prompt.define({ - slug: "raw-input-schema", - // @ts-expect-error input must be a schema function - input: typedPrompt.inputSchema, - render: () => [prompt.user`Nope`], - }); - - prompt.define({ - slug: "raw-output-schema", - input: (s) => s.object({}), - // @ts-expect-error output must be a schema function - output: typedPrompt.outputSchema, - render: () => [prompt.user`Nope`], - }); - prompt.define({ slug: "prompt-output-field", - input: (s) => s.object({}), - output: (s) => + inputSchema: (s) => s.object({}), + outputSchema: (s) => s.object({ // @ts-expect-error output schema helpers do not include prompt helpers prompt: s.builtMessagesPrompt(), }), - render: () => [prompt.user`Nope`], + template: () => [prompt.user`Nope`], }); prompt.define({ slug: "prompt-output-array", - input: (s) => s.object({}), - output: (s) => + inputSchema: (s) => s.object({}), + outputSchema: (s) => // @ts-expect-error output schema helpers do not include prompt helpers s.array(s.builtStringPrompt()), - render: () => [prompt.user`Nope`], + template: () => [prompt.user`Nope`], }); } - - expect(() => - prompt.define({ - slug: "runtime-raw-input-schema", - input: typedPrompt.inputSchema as never, - render: () => [prompt.user`Nope`], - }), - ).toThrow("input must be a schema function"); - expect(() => - prompt.define({ - slug: "runtime-raw-output-schema", - input: (s) => s.object({}), - output: typedPrompt.outputSchema as never, - render: () => [prompt.user`Nope`], - }), - ).toThrow("output must be a schema function"); }); test("passes flat prompt snapshots to custom adapters", () => { const classify = prompt.define({ slug: "classify", model: "gpt-4o-mini", - input: (s) => + inputSchema: (s) => s.object({ text: s.string(), }), - output: (s) => + outputSchema: (s) => s.object({ label: s.enum(["bug", "question"]), }), - render: ({ variables }) => [prompt.user`Classify: ${variables.text}`], + template: ({ variables }) => [prompt.user`Classify: ${variables.text}`], }); const summarize = prompt.define({ slug: "summarize", - input: (s) => + inputSchema: (s) => s.object({ text: s.string(), }), - render: ({ variables }) => prompt.text`Summarize: ${variables.text}`, + template: ({ variables }) => prompt.text`Summarize: ${variables.text}`, }); expect( @@ -1158,7 +1104,7 @@ describe("experimental prompt API", () => { test("input schema helper exposes only the explicit built prompt helpers", () => { prompt.define({ slug: "input-helper-surface", - input: (s) => { + inputSchema: (s) => { expect("builtMessagesPrompt" in s).toBe(true); expect("builtStringPrompt" in s).toBe(true); expect("messagesPromptDefinition" in s).toBe(true); @@ -1189,7 +1135,7 @@ describe("experimental prompt API", () => { return s.object({}); }, - render: () => [prompt.user`Ok`], + template: () => [prompt.user`Ok`], }); }); }); diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts index 4b7ff4df8..729cf1ac6 100644 --- a/js/src/experimental-prompt-api.ts +++ b/js/src/experimental-prompt-api.ts @@ -299,11 +299,10 @@ export type ExperimentalPromptData = { } ); -type PromptRenderResult = readonly PromptMessage[] | PromptText; +type PromptTemplateResult = readonly PromptMessage[] | PromptText; -type PromptRenderContext = { +type PromptTemplateContext = { variables: TVariables; - values: TValues; include: ( definition: TDefinition, input: InputOf, @@ -346,27 +345,26 @@ type OutputSchemaHelpers = typeof outputSchemaHelpers; type PromptDefinitionOptions< TInputSchema extends InputSchema, TOutputSchema extends OutputSchema | undefined, - TRenderResult extends PromptRenderResult, + TTemplateResult extends PromptTemplateResult, > = { id?: string; slug: string; name?: string; version?: string; model?: string; - input: (s: InputSchemaHelpers) => TInputSchema; - output?: (s: OutputSchemaHelpers) => TOutputSchema; - render: ( - context: PromptRenderContext< - PromptTemplateField>, - InferSchema + inputSchema: (s: InputSchemaHelpers) => TInputSchema; + outputSchema?: (s: OutputSchemaHelpers) => TOutputSchema; + template: ( + context: PromptTemplateContext< + PromptTemplateField> >, - ) => TRenderResult; + ) => TTemplateResult; }; class PromptDefinition< TInputSchema extends InputSchema, TOutputSchema extends OutputSchema | undefined, - TRenderResult extends PromptRenderResult, + TTemplateResult extends PromptTemplateResult, > { readonly [promptDefinitionMarker] = true; readonly id?: string; @@ -377,37 +375,31 @@ class PromptDefinition< readonly inputSchema: TInputSchema; readonly outputSchema?: TOutputSchema; - private readonly renderer: PromptDefinitionOptions< + private readonly template: PromptDefinitionOptions< TInputSchema, TOutputSchema, - TRenderResult - >["render"]; + TTemplateResult + >["template"]; constructor( - opts: PromptDefinitionOptions, + opts: PromptDefinitionOptions, ) { this.id = opts.id; this.slug = opts.slug; this.name = opts.name; this.version = opts.version; this.model = opts.model; - if (typeof opts.input !== "function") { - throw new Error("input must be a schema function"); - } - if (opts.output !== undefined && typeof opts.output !== "function") { - throw new Error("output must be a schema function"); - } - this.inputSchema = opts.input(inputSchemaHelpers); - this.outputSchema = opts.output?.(outputSchemaHelpers); - this.renderer = opts.render; + this.inputSchema = opts.inputSchema(inputSchemaHelpers); + this.outputSchema = opts.outputSchema?.(outputSchemaHelpers); + this.template = opts.template; } build( input: InferInputSchema, - ): BuiltPromptForRenderResult< + ): BuiltPromptForTemplateResult< InferSchema, InferOutput, - TRenderResult + TTemplateResult > { const parsedInput = this.inputSchema.parse( input, @@ -423,9 +415,8 @@ class PromptDefinition< throw new Error("prompt variables could not resolve a built prompt"); }, ) as PromptTemplateField>; - const rendered = this.renderer({ + const rendered = this.template({ variables, - values: parsedInput, include: (definition, includeInput) => definition.build(includeInput) as BuiltPromptOf, }); @@ -471,19 +462,19 @@ class PromptDefinition< input: parsedInput, content: rendered.content, dependencies, - }) as BuiltPromptForRenderResult< + }) as BuiltPromptForTemplateResult< InferSchema, InferOutput, - TRenderResult + TTemplateResult >; } if (!Array.isArray(rendered)) { - throw new Error("render must return a message array or prompt.text"); + throw new Error("template must return a message array or prompt.text"); } const messages = rendered.map((message, index) => - assertPromptMessage(message, `render[${index}]`), + assertPromptMessage(message, `template[${index}]`), ); const dependencies = createPromptDependencies( root, @@ -514,10 +505,10 @@ class PromptDefinition< input: parsedInput, messages, dependencies, - }) as BuiltPromptForRenderResult< + }) as BuiltPromptForTemplateResult< InferSchema, InferOutput, - TRenderResult + TTemplateResult >; } @@ -542,9 +533,8 @@ class PromptDefinition< return templateDataToBuiltPrompt(nested, kind); }, ) as PromptTemplateField>; - const rendered = this.renderer({ + const rendered = this.template({ variables, - values: createUnavailableValuesProxy() as InferSchema, include: (definition) => { const nested = definition.compileTemplate( createRootTemplateScope(definition.inputSchema), @@ -578,11 +568,11 @@ class PromptDefinition< } if (!Array.isArray(rendered)) { - throw new Error("render must return a message array or prompt.text"); + throw new Error("template must return a message array or prompt.text"); } const messages = rendered.map((message, index) => - assertPromptMessage(message, `render[${index}]`), + assertPromptMessage(message, `template[${index}]`), ); const dependencies = createPromptDependencies(root, inputSnapshot, [ ...collectBuiltPromptDependencies(variables, this.slug), @@ -604,7 +594,7 @@ class PromptDefinition< type AnyPromptDefinition = PromptDefinition< InputSchema, OutputSchema | undefined, - PromptRenderResult + PromptTemplateResult >; type AnyMessagesPromptDefinition = PromptDefinition< @@ -628,7 +618,7 @@ type InputOf = TDefinition extends PromptDefinition< infer TInputSchema, OutputSchema | undefined, - PromptRenderResult + PromptTemplateResult > ? InferInputSchema : never; @@ -637,7 +627,7 @@ type ParsedInputOf = TDefinition extends PromptDefinition< infer TInputSchema, OutputSchema | undefined, - PromptRenderResult + PromptTemplateResult > ? InferSchema : never; @@ -646,7 +636,7 @@ type OutputOf = TDefinition extends PromptDefinition< InputSchema, infer TOutputSchema, - PromptRenderResult + PromptTemplateResult > ? InferOutput : never; @@ -655,19 +645,19 @@ type BuiltPromptOf = TDefinition extends PromptDefinition< infer TInputSchema, infer TOutputSchema, - infer TRenderResult + infer TTemplateResult > - ? BuiltPromptForRenderResult< + ? BuiltPromptForTemplateResult< InferSchema, InferOutput, - TRenderResult + TTemplateResult > : never; -type BuiltPromptForRenderResult = - TRenderResult extends PromptText +type BuiltPromptForTemplateResult = + TTemplateResult extends PromptText ? BuiltStringPrompt - : TRenderResult extends readonly PromptMessage[] + : TTemplateResult extends readonly PromptMessage[] ? BuiltMessagesPrompt : never; @@ -941,10 +931,10 @@ type AnyBuiltPrompt = function definePrompt< TInputSchema extends InputSchema, TOutputSchema extends OutputSchema | undefined = undefined, - TRenderResult extends PromptRenderResult = PromptRenderResult, + TTemplateResult extends PromptTemplateResult = PromptTemplateResult, >( - opts: PromptDefinitionOptions, -): PromptDefinition { + opts: PromptDefinitionOptions, +): PromptDefinition { return new PromptDefinition(opts); } @@ -1509,22 +1499,6 @@ function stringifyRuntimeValue(value: unknown): string { return JSON.stringify(value); } -function createUnavailableValuesProxy(): unknown { - return new Proxy( - {}, - { - get(_target, key) { - if (typeof key === "symbol") { - return undefined; - } - throw new Error( - "Runtime values are not available while exporting prompt data; use variables in prompt templates.", - ); - }, - }, - ); -} - function isMustacheTemplateValue( value: unknown, ): value is MustacheTemplateValue { From 9fd0a04782f92ea0c46fae2026407027afe72981 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Mon, 6 Jul 2026 10:11:31 +0200 Subject: [PATCH 6/6] simplifications structure tests --- js/src/experimental-prompt-adapters.ts | 428 ------------- js/src/experimental-prompt-adapters/ai-sdk.ts | 90 +++ js/src/experimental-prompt-adapters/index.ts | 9 + js/src/experimental-prompt-adapters/openai.ts | 128 ++++ js/src/experimental-prompt-adapters/types.ts | 21 + js/src/experimental-prompt-adapters/utils.ts | 202 +++++++ .../experimental-prompt-api-schema-utils.ts | 366 +++++++++++ js/src/experimental-prompt-api.ts | 572 +++--------------- js/src/experimental-prompt-api.typecheck.ts | 303 ++++++++++ js/src/template-generators/mustache.ts | 117 ++++ 10 files changed, 1320 insertions(+), 916 deletions(-) delete mode 100644 js/src/experimental-prompt-adapters.ts create mode 100644 js/src/experimental-prompt-adapters/ai-sdk.ts create mode 100644 js/src/experimental-prompt-adapters/index.ts create mode 100644 js/src/experimental-prompt-adapters/openai.ts create mode 100644 js/src/experimental-prompt-adapters/types.ts create mode 100644 js/src/experimental-prompt-adapters/utils.ts create mode 100644 js/src/experimental-prompt-api-schema-utils.ts create mode 100644 js/src/experimental-prompt-api.typecheck.ts create mode 100644 js/src/template-generators/mustache.ts diff --git a/js/src/experimental-prompt-adapters.ts b/js/src/experimental-prompt-adapters.ts deleted file mode 100644 index 5bd31ca5d..000000000 --- a/js/src/experimental-prompt-adapters.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { BaseAttachment, ReadonlyAttachment } from "./logger"; -import type { BraintrustState } from "./logger"; -import type { AttachmentReferenceType as AttachmentReference } from "./generated_types"; -import type { - InlineAttachmentReference, - PromptDependencies, - PromptJsonSchema, - PromptMessage, - PromptMessageContentPart, -} from "./experimental-prompt-api"; - -type PromptAdapterInput = { - kind: "messages" | "string"; - model?: string; - inputSchema: { toJSONSchema(): PromptJsonSchema }; - outputSchema?: { toJSONSchema(): PromptJsonSchema }; - input: unknown; - messages: PromptMessage[]; - content?: string; - dependencies: PromptDependencies; -}; - -type OpenAIContentPart = - | { type: "text"; text: string } - | { - type: "image_url"; - image_url: { url: string; detail?: "auto" | "low" | "high" }; - } - | { - type: "file"; - file: { file_data?: string; file_id?: string; filename?: string }; - }; - -type OpenAIChatMessage = Omit & { - content: string | OpenAIContentPart[]; -}; - -type OpenAIChatPromptArgs = { - model?: string; - messages: OpenAIChatMessage[]; - response_format?: { - type: "json_schema"; - json_schema: { - name: string; - schema: PromptJsonSchema; - strict: true; - }; - }; - span_info: { - metadata: { - prompt: PromptDependencies; - }; - }; -}; - -type AISDKContentPart = - | { type: "text"; text: string } - | { type: "image"; image: string; mediaType?: string } - | { type: "file"; data: string; mediaType: string; filename?: string }; - -type AISDKMessage = Omit & { - content: string | AISDKContentPart[]; -}; - -type AISDKGenerateObjectPromptArgs = { - model?: string; - messages: AISDKMessage[]; - schema?: PromptJsonSchema; - experimental_telemetry: { - metadata: { - braintrustPrompt: PromptDependencies; - }; - }; -}; - -type AdapterOptions = { - state?: BraintrustState; -}; - -type OpenAIChatAdapterOptions = AdapterOptions; -type AISDKGenerateObjectAdapterOptions = AdapterOptions; - -type ResolvedPromptFile = { - data: string; - contentType?: string; - filename?: string; - detail?: "auto" | "low" | "high"; -}; - -function schemaName(slug: string): string { - const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); - return name.length > 0 ? `${name}_output` : "prompt_output"; -} - -function openAIChatAdapter( - options: OpenAIChatAdapterOptions = {}, -): (builtPrompt: PromptAdapterInput) => Promise { - return async (builtPrompt) => { - const outputSchema = builtPrompt.outputSchema?.toJSONSchema(); - return { - model: builtPrompt.model, - messages: await Promise.all( - builtPrompt.messages.map((message) => - renderOpenAIMessage(message, options), - ), - ), - ...(outputSchema - ? { - response_format: { - type: "json_schema" as const, - json_schema: { - name: schemaName(builtPrompt.dependencies.root.slug), - schema: outputSchema, - strict: true as const, - }, - }, - } - : undefined), - span_info: { - metadata: { - prompt: builtPrompt.dependencies, - }, - }, - }; - }; -} - -async function renderOpenAIMessage( - message: PromptMessage, - options: AdapterOptions, -): Promise { - if (typeof message.content === "string") { - return message as OpenAIChatMessage; - } - return { - ...message, - content: await Promise.all( - message.content.map((part) => renderOpenAIContentPart(part, options)), - ), - }; -} - -async function renderOpenAIContentPart( - part: PromptMessageContentPart, - options: AdapterOptions, -): Promise { - if (part.type === "text") { - return part; - } - - const resolved = await resolvePromptFile(part, options); - if (isImageFile(resolved)) { - return { - type: "image_url", - image_url: { - url: resolved.data, - ...(resolved.detail ? { detail: resolved.detail } : undefined), - }, - }; - } - - return { - type: "file", - file: isLikelyFileId(resolved.data) - ? { - file_id: resolved.data, - ...(resolved.filename ? { filename: resolved.filename } : undefined), - } - : { - file_data: resolved.data, - ...(resolved.filename ? { filename: resolved.filename } : undefined), - }, - }; -} - -function aiSDKGenerateObjectAdapter( - options: AISDKGenerateObjectAdapterOptions = {}, -): (builtPrompt: PromptAdapterInput) => Promise { - return async (builtPrompt) => ({ - model: builtPrompt.model, - messages: await Promise.all( - builtPrompt.messages.map((message) => - renderAISDKMessage(message, options), - ), - ), - schema: builtPrompt.outputSchema?.toJSONSchema(), - experimental_telemetry: { - metadata: { - braintrustPrompt: builtPrompt.dependencies, - }, - }, - }); -} - -async function renderAISDKMessage( - message: PromptMessage, - options: AdapterOptions, -): Promise { - if (typeof message.content === "string") { - return message as AISDKMessage; - } - return { - ...message, - content: await Promise.all( - message.content.map((part) => renderAISDKContentPart(part, options)), - ), - }; -} - -async function renderAISDKContentPart( - part: PromptMessageContentPart, - options: AdapterOptions, -): Promise { - if (part.type === "text") { - return part; - } - - const resolved = await resolvePromptFile(part, options); - if (isImageFile(resolved)) { - return { - type: "image", - image: resolved.data, - ...(resolved.contentType - ? { mediaType: resolved.contentType } - : undefined), - }; - } - - return { - type: "file", - data: resolved.data, - mediaType: resolved.contentType ?? "application/octet-stream", - ...(resolved.filename ? { filename: resolved.filename } : undefined), - }; -} - -async function resolvePromptFile( - part: Extract, - options: AdapterOptions, -): Promise { - const value = part.file.value; - const optionContentType = part.file.contentType; - const optionFilename = part.file.filename; - - if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { - const reference = value.reference; - const data = - value instanceof ReadonlyAttachment - ? await value.asBase64Url() - : await blobToDataUrl(await value.data(), reference.content_type); - return { - data, - contentType: optionContentType ?? reference.content_type, - filename: optionFilename ?? reference.filename, - detail: part.file.detail, - }; - } - - if (isAttachmentReference(value)) { - const data = await new ReadonlyAttachment( - value, - options.state, - ).asBase64Url(); - return { - data, - contentType: optionContentType ?? value.content_type, - filename: optionFilename ?? value.filename, - detail: part.file.detail, - }; - } - - if (isInlineAttachmentReference(value)) { - const data = value.data ?? value.src; - return { - data, - contentType: - optionContentType ?? value.content_type ?? contentTypeFromString(data), - filename: optionFilename ?? value.filename, - detail: part.file.detail, - }; - } - - if (isBlob(value)) { - const contentType = optionContentType ?? (value.type || undefined); - return { - data: await blobToDataUrl(value, contentType), - contentType, - filename: optionFilename, - detail: part.file.detail, - }; - } - - if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) { - const contentType = optionContentType ?? "application/octet-stream"; - return { - data: await blobToDataUrl( - new Blob([value as BlobPart], { type: contentType }), - contentType, - ), - contentType, - filename: optionFilename, - detail: part.file.detail, - }; - } - - if (typeof value === "string") { - return { - data: value, - contentType: optionContentType ?? contentTypeFromString(value), - filename: optionFilename, - detail: part.file.detail, - }; - } - - throw new Error("prompt.file value must be an attachment-compatible value"); -} - -function isImageFile(file: ResolvedPromptFile): boolean { - if (file.contentType?.startsWith("image/")) { - return true; - } - if (file.contentType && !file.contentType.startsWith("image/")) { - return false; - } - return isHttpUrl(file.data); -} - -function isLikelyFileId(value: string): boolean { - return !isHttpUrl(value) && !value.startsWith("data:"); -} - -function contentTypeFromString(value: string): string | undefined { - const dataUrlContentType = value.match(/^data:([^;,]+)[;,]/)?.[1]; - if (dataUrlContentType) { - return dataUrlContentType; - } - - const extension = value.split(/[?#]/, 1)[0]?.split(".").at(-1)?.toLowerCase(); - switch (extension) { - case "png": - return "image/png"; - case "jpg": - case "jpeg": - return "image/jpeg"; - case "gif": - return "image/gif"; - case "webp": - return "image/webp"; - case "pdf": - return "application/pdf"; - case "txt": - return "text/plain"; - case "json": - return "application/json"; - default: - return undefined; - } -} - -function isHttpUrl(value: string): boolean { - try { - const url = new URL(value); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - -function isBlob(value: unknown): value is Blob { - return typeof Blob !== "undefined" && value instanceof Blob; -} - -function isAttachmentReference(value: unknown): value is AttachmentReference { - return ( - isRecord(value) && - ((value.type === "braintrust_attachment" && - typeof value.key === "string" && - typeof value.filename === "string" && - typeof value.content_type === "string") || - (value.type === "external_attachment" && - typeof value.url === "string" && - typeof value.filename === "string" && - typeof value.content_type === "string")) - ); -} - -function isInlineAttachmentReference( - value: unknown, -): value is InlineAttachmentReference { - return ( - isRecord(value) && - value.type === "inline_attachment" && - typeof value.src === "string" && - (value.content_type === undefined || - typeof value.content_type === "string") && - (value.filename === undefined || typeof value.filename === "string") && - (value.data === undefined || typeof value.data === "string") - ); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -async function blobToDataUrl( - blob: Blob, - contentType?: string, -): Promise { - const buffer = await blob.arrayBuffer(); - const base64 = - typeof Buffer !== "undefined" - ? Buffer.from(buffer).toString("base64") - : bytesToBase64(new Uint8Array(buffer)); - return `data:${contentType ?? (blob.type || "application/octet-stream")};base64,${base64}`; -} - -function bytesToBase64(bytes: Uint8Array): string { - let binary = ""; - for (const byte of bytes) { - binary += String.fromCharCode(byte); - } - return btoa(binary); -} - -export const adapters = { - openAIChat: openAIChatAdapter, - aiSDKGenerateObject: aiSDKGenerateObjectAdapter, -}; diff --git a/js/src/experimental-prompt-adapters/ai-sdk.ts b/js/src/experimental-prompt-adapters/ai-sdk.ts new file mode 100644 index 000000000..eb16384ec --- /dev/null +++ b/js/src/experimental-prompt-adapters/ai-sdk.ts @@ -0,0 +1,90 @@ +import type { + PromptMessage, + PromptMessageContentPart, +} from "../experimental-prompt-api"; +import type { PromptJsonSchema } from "../experimental-prompt-api-schema-utils"; +import { isImageFile, resolvePromptFile } from "./utils"; +import type { AdapterOptions, PromptAdapterInput } from "./types"; + +type AISDKContentPart = + | { type: "text"; text: string } + | { type: "image"; image: string; mediaType?: string } + | { type: "file"; data: string; mediaType: string; filename?: string }; + +type AISDKMessage = Omit & { + content: string | AISDKContentPart[]; +}; + +type AISDKGenerateObjectPromptArgs = { + model?: string; + messages: AISDKMessage[]; + schema?: PromptJsonSchema; + experimental_telemetry: { + metadata: { + braintrustPrompt: PromptAdapterInput["dependencies"]; + }; + }; +}; + +type AISDKGenerateObjectAdapterOptions = AdapterOptions; + +export function aiSDKGenerateObjectAdapter( + options: AISDKGenerateObjectAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => ({ + model: builtPrompt.model, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderAISDKMessage(message, options), + ), + ), + schema: builtPrompt.outputSchema?.toJSONSchema(), + experimental_telemetry: { + metadata: { + braintrustPrompt: builtPrompt.dependencies, + }, + }, + }); +} + +async function renderAISDKMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as AISDKMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderAISDKContentPart(part, options)), + ), + }; +} + +async function renderAISDKContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image", + image: resolved.data, + ...(resolved.contentType + ? { mediaType: resolved.contentType } + : undefined), + }; + } + + return { + type: "file", + data: resolved.data, + mediaType: resolved.contentType ?? "application/octet-stream", + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }; +} diff --git a/js/src/experimental-prompt-adapters/index.ts b/js/src/experimental-prompt-adapters/index.ts new file mode 100644 index 000000000..5edafb6ff --- /dev/null +++ b/js/src/experimental-prompt-adapters/index.ts @@ -0,0 +1,9 @@ +import { aiSDKGenerateObjectAdapter } from "./ai-sdk"; +import { openAIChatAdapter } from "./openai"; + +export { aiSDKGenerateObjectAdapter, openAIChatAdapter }; + +export const adapters = { + openAIChat: openAIChatAdapter, + aiSDKGenerateObject: aiSDKGenerateObjectAdapter, +}; diff --git a/js/src/experimental-prompt-adapters/openai.ts b/js/src/experimental-prompt-adapters/openai.ts new file mode 100644 index 000000000..c597a0eb6 --- /dev/null +++ b/js/src/experimental-prompt-adapters/openai.ts @@ -0,0 +1,128 @@ +import type { + PromptMessage, + PromptMessageContentPart, +} from "../experimental-prompt-api"; +import type { PromptJsonSchema } from "../experimental-prompt-api-schema-utils"; +import type { AdapterOptions, PromptAdapterInput } from "./types"; +import { isImageFile, isLikelyFileId, resolvePromptFile } from "./utils"; + +type OpenAIContentPart = + | { type: "text"; text: string } + | { + type: "image_url"; + image_url: { url: string; detail?: "auto" | "low" | "high" }; + } + | { + type: "file"; + file: { file_data?: string; file_id?: string; filename?: string }; + }; + +type OpenAIChatMessage = Omit & { + content: string | OpenAIContentPart[]; +}; + +type OpenAIChatPromptArgs = { + model?: string; + messages: OpenAIChatMessage[]; + response_format?: { + type: "json_schema"; + json_schema: { + name: string; + schema: PromptJsonSchema; + strict: true; + }; + }; + span_info: { + metadata: { + prompt: PromptAdapterInput["dependencies"]; + }; + }; +}; + +type OpenAIChatAdapterOptions = AdapterOptions; + +function schemaName(slug: string): string { + const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); + return name.length > 0 ? `${name}_output` : "prompt_output"; +} + +export function openAIChatAdapter( + options: OpenAIChatAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => { + const outputSchema = builtPrompt.outputSchema?.toJSONSchema(); + return { + model: builtPrompt.model, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderOpenAIMessage(message, options), + ), + ), + ...(outputSchema + ? { + response_format: { + type: "json_schema" as const, + json_schema: { + name: schemaName(builtPrompt.dependencies.root.slug), + schema: outputSchema, + strict: true as const, + }, + }, + } + : undefined), + span_info: { + metadata: { + prompt: builtPrompt.dependencies, + }, + }, + }; + }; +} + +async function renderOpenAIMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as OpenAIChatMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderOpenAIContentPart(part, options)), + ), + }; +} + +async function renderOpenAIContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image_url", + image_url: { + url: resolved.data, + ...(resolved.detail ? { detail: resolved.detail } : undefined), + }, + }; + } + + return { + type: "file", + file: isLikelyFileId(resolved.data) + ? { + file_id: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + } + : { + file_data: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }, + }; +} diff --git a/js/src/experimental-prompt-adapters/types.ts b/js/src/experimental-prompt-adapters/types.ts new file mode 100644 index 000000000..49f60c3b6 --- /dev/null +++ b/js/src/experimental-prompt-adapters/types.ts @@ -0,0 +1,21 @@ +import type { PromptJsonSchema } from "../experimental-prompt-api-schema-utils"; +import type { + PromptDependencies, + PromptMessage, +} from "../experimental-prompt-api"; +import type { BraintrustState } from "../logger"; + +export type PromptAdapterInput = { + kind: "messages" | "string"; + model?: string; + inputSchema: { toJSONSchema(): PromptJsonSchema }; + outputSchema?: { toJSONSchema(): PromptJsonSchema }; + input: unknown; + messages: PromptMessage[]; + content?: string; + dependencies: PromptDependencies; +}; + +export type AdapterOptions = { + state?: BraintrustState; +}; diff --git a/js/src/experimental-prompt-adapters/utils.ts b/js/src/experimental-prompt-adapters/utils.ts new file mode 100644 index 000000000..973bcd592 --- /dev/null +++ b/js/src/experimental-prompt-adapters/utils.ts @@ -0,0 +1,202 @@ +import { BaseAttachment, ReadonlyAttachment } from "../logger"; +import type { AttachmentReferenceType as AttachmentReference } from "../generated_types"; +import type { + InlineAttachmentReference, + PromptMessageContentPart, +} from "../experimental-prompt-api"; +import type { AdapterOptions } from "./types"; + +export type ResolvedPromptFile = { + data: string; + contentType?: string; + filename?: string; + detail?: "auto" | "low" | "high"; +}; + +export async function resolvePromptFile( + part: Extract, + options: AdapterOptions, +): Promise { + const value = part.file.value; + const optionContentType = part.file.contentType; + const optionFilename = part.file.filename; + + if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { + const reference = value.reference; + const data = + value instanceof ReadonlyAttachment + ? await value.asBase64Url() + : await blobToDataUrl(await value.data(), reference.content_type); + return { + data, + contentType: optionContentType ?? reference.content_type, + filename: optionFilename ?? reference.filename, + detail: part.file.detail, + }; + } + + if (isAttachmentReference(value)) { + const data = await new ReadonlyAttachment( + value, + options.state, + ).asBase64Url(); + return { + data, + contentType: optionContentType ?? value.content_type, + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isInlineAttachmentReference(value)) { + const data = value.data ?? value.src; + return { + data, + contentType: + optionContentType ?? value.content_type ?? contentTypeFromString(data), + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isBlob(value)) { + const contentType = optionContentType ?? (value.type || undefined); + return { + data: await blobToDataUrl(value, contentType), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) { + const contentType = optionContentType ?? "application/octet-stream"; + return { + data: await blobToDataUrl( + new Blob([value as BlobPart], { type: contentType }), + contentType, + ), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (typeof value === "string") { + return { + data: value, + contentType: optionContentType ?? contentTypeFromString(value), + filename: optionFilename, + detail: part.file.detail, + }; + } + + throw new Error("prompt.file value must be an attachment-compatible value"); +} + +export function isImageFile(file: ResolvedPromptFile): boolean { + if (file.contentType?.startsWith("image/")) { + return true; + } + if (file.contentType && !file.contentType.startsWith("image/")) { + return false; + } + return isHttpUrl(file.data); +} + +export function isLikelyFileId(value: string): boolean { + return !isHttpUrl(value) && !value.startsWith("data:"); +} + +function contentTypeFromString(value: string): string | undefined { + const dataUrlContentType = value.match(/^data:([^;,]+)[;,]/)?.[1]; + if (dataUrlContentType) { + return dataUrlContentType; + } + + const extension = value.split(/[?#]/, 1)[0]?.split(".").at(-1)?.toLowerCase(); + switch (extension) { + case "png": + return "image/png"; + case "jpg": + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "pdf": + return "application/pdf"; + case "txt": + return "text/plain"; + case "json": + return "application/json"; + default: + return undefined; + } +} + +function isHttpUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== "undefined" && value instanceof Blob; +} + +function isAttachmentReference(value: unknown): value is AttachmentReference { + return ( + isRecord(value) && + ((value.type === "braintrust_attachment" && + typeof value.key === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string") || + (value.type === "external_attachment" && + typeof value.url === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string")) + ); +} + +function isInlineAttachmentReference( + value: unknown, +): value is InlineAttachmentReference { + return ( + isRecord(value) && + value.type === "inline_attachment" && + typeof value.src === "string" && + (value.content_type === undefined || + typeof value.content_type === "string") && + (value.filename === undefined || typeof value.filename === "string") && + (value.data === undefined || typeof value.data === "string") + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function blobToDataUrl( + blob: Blob, + contentType?: string, +): Promise { + const buffer = await blob.arrayBuffer(); + const base64 = + typeof Buffer !== "undefined" + ? Buffer.from(buffer).toString("base64") + : bytesToBase64(new Uint8Array(buffer)); + return `data:${contentType ?? (blob.type || "application/octet-stream")};base64,${base64}`; +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + return btoa(binary); +} diff --git a/js/src/experimental-prompt-api-schema-utils.ts b/js/src/experimental-prompt-api-schema-utils.ts new file mode 100644 index 000000000..7bfd6f232 --- /dev/null +++ b/js/src/experimental-prompt-api-schema-utils.ts @@ -0,0 +1,366 @@ +type JsonPrimitive = string | number | boolean | null; + +export type PromptJsonSchema = { + type?: string; + properties?: Record; + required?: string[]; + items?: PromptJsonSchema; + enum?: JsonPrimitive[]; + additionalProperties?: boolean; + description?: string; + "x-bt-type"?: string; +}; + +type SchemaParser = (value: unknown, path: string, root: unknown) => T; + +export type SchemaDomain = "input" | "output"; +export type PromptKind = "messages" | "string"; + +export type PromptSchemaTemplateInfo = + | { + type: "object"; + shape: SchemaShape; + } + | { + type: "array"; + item: AnySchema; + } + | { + type: "promptDefinition"; + definition: unknown; + kind: PromptKind; + } + | { + type: "attachment"; + }; + +export class PromptSchema< + TParsed, + TInput = TParsed, + TKind = unknown, + TDomain extends SchemaDomain = "input", +> { + readonly _type!: TParsed; + readonly _input!: TInput; + readonly _kind!: TKind; + readonly _domain!: TDomain; + + constructor( + private readonly parser: SchemaParser, + private readonly jsonSchema: () => PromptJsonSchema, + public readonly isOptional = false, + public readonly templateInfo?: PromptSchemaTemplateInfo, + ) {} + + parse(value: unknown, path = "value", root: unknown = value): TParsed { + return this.parser(value, path, root); + } + + toJSONSchema(): PromptJsonSchema { + return this.jsonSchema(); + } + + optional(): PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + > { + return new PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + >( + (value, path, root) => + value === undefined ? undefined : this.parser(value, path, root), + () => this.jsonSchema(), + true, + this.templateInfo, + ); + } +} + +export type AnySchema = PromptSchema; +export type InputSchema = PromptSchema; +export type OutputSchema = PromptSchema; + +export type InferSchema = + TSchema extends PromptSchema + ? TParsed + : never; + +export type InferInputSchema = + TSchema extends PromptSchema + ? TInput + : never; + +export type SchemaShape = Record; +export type InputSchemaShape = Record; +export type OutputSchemaShape = Record; + +type OptionalParsedKeys = { + [K in keyof TShape]: undefined extends InferSchema ? K : never; +}[keyof TShape]; + +type OptionalInputKeys = { + [K in keyof TShape]: undefined extends InferObjectInputSchema< + TShape[K], + TShape, + K + > + ? K + : never; +}[keyof TShape]; + +type InferParsedObject = { + [K in keyof TShape as K extends OptionalParsedKeys + ? never + : K]: InferSchema; +} & { + [K in OptionalParsedKeys]?: Exclude< + InferSchema, + undefined + >; +}; + +type InferInputObject = { + [K in keyof TShape as K extends OptionalInputKeys + ? never + : K]: InferObjectInputSchema; +} & { + [K in OptionalInputKeys]?: Exclude< + InferObjectInputSchema, + undefined + >; +}; + +export type PromptFieldKind< + TBuiltPrompt, + TPromptInput, + TPromptKind extends PromptKind, +> = { + type: "prompt"; + builtPrompt: TBuiltPrompt; + promptInput: TPromptInput; + promptKind: TPromptKind; +}; + +type PromptInputOverrides< + TInput, + TParentKeys extends PropertyKey, +> = TInput extends object + ? Omit & + Partial>> + : TInput; + +type InferObjectInputSchema< + TSchema extends AnySchema, + TShape extends SchemaShape, + TKey extends keyof TShape, +> = + TSchema extends PromptSchema + ? TKind extends PromptFieldKind< + infer TBuiltPrompt, + infer TPromptInput, + PromptKind + > + ? + | TBuiltPrompt + | PromptInputOverrides> + | (undefined extends TInput ? undefined : never) + : TInput + : never; + +export function stringSchema< + TDomain extends SchemaDomain = "input", +>(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string") { + throw new Error(`${path} must be a string`); + } + return value; + }, + () => ({ type: "string" }), + ); +} + +export function numberSchema< + TDomain extends SchemaDomain = "input", +>(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "number") { + throw new Error(`${path} must be a number`); + } + return value; + }, + () => ({ type: "number" }), + ); +} + +export function booleanSchema< + TDomain extends SchemaDomain = "input", +>(): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "boolean") { + throw new Error(`${path} must be a boolean`); + } + return value; + }, + () => ({ type: "boolean" }), + ); +} + +export function enumSchema< + const TValues extends readonly [string, ...string[]], + TDomain extends SchemaDomain = "input", +>( + values: TValues, +): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string" || !values.includes(value)) { + throw new Error(`${path} must be one of ${values.join(", ")}`); + } + return value; + }, + () => ({ type: "string", enum: [...values] }), + ); +} + +function createArraySchema< + TItemSchema extends AnySchema, + TDomain extends SchemaDomain, +>( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain +> { + return new PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain + >( + (value, path, root) => { + if (!Array.isArray(value)) { + throw new Error(`${path} must be an array`); + } + return value.map((itemValue, index) => + item.parse(itemValue, `${path}[${index}]`, root), + ) as InferSchema[]; + }, + () => ({ type: "array", items: item.toJSONSchema() }), + false, + { type: "array", item }, + ); +} + +export function arraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "input" +> { + return createArraySchema(item); +} + +export function outputArraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "output" +> { + return createArraySchema(item); +} + +function createObjectSchema< + TShape extends SchemaShape, + TDomain extends SchemaDomain, +>( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain +> { + return new PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain + >( + (value, path, root) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + const record = value as Record; + const rootInput = root === undefined ? record : root; + return Object.fromEntries( + Object.entries(shape) + .filter(([key, schema]) => key in record || !schema.isOptional) + .map(([key, schema]) => [ + key, + schema.parse(record[key], `${path}.${key}`, rootInput), + ]), + ) as InferParsedObject; + }, + () => ({ + type: "object", + properties: Object.fromEntries( + Object.entries(shape).map(([key, schema]) => [ + key, + schema.toJSONSchema(), + ]), + ), + required: Object.entries(shape) + .filter(([, schema]) => !schema.isOptional) + .map(([key]) => key), + additionalProperties: false, + }), + false, + { type: "object", shape }, + ); +} + +export function objectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "input" +> { + return createObjectSchema(shape); +} + +export function outputObjectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "output" +> { + return createObjectSchema(shape); +} + +export function unknownSchema< + TDomain extends SchemaDomain = "input", +>(): PromptSchema { + return new PromptSchema( + (value) => value, + () => ({}), + ); +} diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts index 729cf1ac6..0818dcf81 100644 --- a/js/src/experimental-prompt-api.ts +++ b/js/src/experimental-prompt-api.ts @@ -1,165 +1,31 @@ import { adapters } from "./experimental-prompt-adapters"; +import { + PromptSchema, + arraySchema, + booleanSchema, + enumSchema, + numberSchema, + objectSchema, + outputArraySchema, + outputObjectSchema, + stringSchema, + unknownSchema, +} from "./experimental-prompt-api-schema-utils"; import { BaseAttachment, ReadonlyAttachment } from "./logger"; +import type { AttachmentReferenceType as AttachmentReference } from "./generated_types"; import type { - AttachmentReferenceType as AttachmentReference, - ChatCompletionContentPartType, - ChatCompletionMessageParamType, -} from "./generated_types"; -import type { PromptDefinition as MustachePromptDefinition } from "./prompt-schemas"; - -type JsonPrimitive = string | number | boolean | null; - -export type PromptJsonSchema = { - type?: string; - properties?: Record; - required?: string[]; - items?: PromptJsonSchema; - enum?: JsonPrimitive[]; - additionalProperties?: boolean; - description?: string; - "x-bt-type"?: string; -}; - -type SchemaParser = (value: unknown, path: string, root: unknown) => T; - -type SchemaDomain = "input" | "output"; -type PromptKind = "messages" | "string"; - -type PromptSchemaTemplateInfo = - | { - type: "object"; - shape: SchemaShape; - } - | { - type: "array"; - item: AnySchema; - } - | { - type: "promptDefinition"; - definition: AnyPromptDefinition; - kind: PromptKind; - } - | { - type: "attachment"; - }; - -class PromptSchema< - TParsed, - TInput = TParsed, - TKind = unknown, - TDomain extends SchemaDomain = "input", -> { - readonly _type!: TParsed; - readonly _input!: TInput; - readonly _kind!: TKind; - readonly _domain!: TDomain; - - constructor( - private readonly parser: SchemaParser, - private readonly jsonSchema: () => PromptJsonSchema, - public readonly isOptional = false, - public readonly templateInfo?: PromptSchemaTemplateInfo, - ) {} - - parse(value: unknown, path = "value", root: unknown = value): TParsed { - return this.parser(value, path, root); - } - - toJSONSchema(): PromptJsonSchema { - return this.jsonSchema(); - } - - optional(): PromptSchema< - TParsed | undefined, - TInput | undefined, - TKind, - TDomain - > { - return new PromptSchema< - TParsed | undefined, - TInput | undefined, - TKind, - TDomain - >( - (value, path, root) => - value === undefined ? undefined : this.parser(value, path, root), - () => this.jsonSchema(), - true, - this.templateInfo, - ); - } -} - -type AnySchema = PromptSchema; -type InputSchema = PromptSchema; -type OutputSchema = PromptSchema; - -type InferSchema = - TSchema extends PromptSchema - ? TParsed - : never; - -type InferInputSchema = - TSchema extends PromptSchema - ? TInput - : never; - -type SchemaShape = Record; -type InputSchemaShape = Record; -type OutputSchemaShape = Record; - -type OptionalParsedKeys = { - [K in keyof TShape]: undefined extends InferSchema ? K : never; -}[keyof TShape]; - -type OptionalInputKeys = { - [K in keyof TShape]: undefined extends InferObjectInputSchema< - TShape[K], - TShape, - K - > - ? K - : never; -}[keyof TShape]; - -type InferParsedObject = { - [K in keyof TShape as K extends OptionalParsedKeys - ? never - : K]: InferSchema; -} & { - [K in OptionalParsedKeys]?: Exclude< - InferSchema, - undefined - >; -}; - -type InferInputObject = { - [K in keyof TShape as K extends OptionalInputKeys - ? never - : K]: InferObjectInputSchema; -} & { - [K in OptionalInputKeys]?: Exclude< - InferObjectInputSchema, - undefined - >; -}; + InferInputSchema, + InferSchema, + InputSchema, + OutputSchema, + PromptFieldKind as SchemaPromptFieldKind, + PromptJsonSchema, + PromptKind, + SchemaShape, +} from "./experimental-prompt-api-schema-utils"; -type InferObjectInputSchema< - TSchema extends AnySchema, - TShape extends SchemaShape, - TKey extends keyof TShape, -> = - TSchema extends PromptSchema - ? TKind extends PromptFieldKind - ? PromptInputValueForObject< - TDefinition, - TPromptKind, - TShape, - TKey, - TInput - > - : TInput - : never; +export type { PromptJsonSchema } from "./experimental-prompt-api-schema-utils"; +export { promptDefinitionToMustache } from "./template-generators/mustache"; const builtPromptMarker = Symbol("braintrust.experimental_prompt.built"); const promptDefinitionMarker = Symbol( @@ -325,14 +191,19 @@ type PromptListTag = ( ...values: readonly unknown[] ) => PromptText; -type PromptTemplateField = TValue extends AnyBuiltPrompt - ? TValue - : unknown & - (TValue extends readonly (infer TItem)[] - ? { list: PromptListTag & PromptTemplateField } - : TValue extends object - ? { [K in keyof TValue]: PromptTemplateField } - : {}); +type PromptTemplateField = unknown extends TValue + ? unknown + : NonNullable extends AnyBuiltPrompt + ? NonNullable + : NonNullable extends readonly (infer TItem)[] + ? unknown & { list: PromptListTag & PromptTemplateField } + : NonNullable extends object + ? unknown & { + [K in keyof NonNullable]-?: PromptTemplateField< + NonNullable[K] + >; + } + : unknown; type TemplateRenderContext = { sectionPath?: string; @@ -672,19 +543,15 @@ type BuiltPromptForKind< type PromptFieldKind< TDefinition extends AnyPromptDefinition, TPromptKind extends PromptKind, -> = { - type: "prompt"; - definition: TDefinition; - promptKind: TPromptKind; -}; - -type PromptInputOverrides< - TInput, - TParentKeys extends PropertyKey, -> = TInput extends object - ? Omit & - Partial>> - : TInput; +> = SchemaPromptFieldKind< + BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TPromptKind + >, + InputOf, + TPromptKind +>; type PromptInputValue< TDefinition extends AnyPromptDefinition, @@ -697,21 +564,6 @@ type PromptInputValue< > | InputOf; -type PromptInputValueForObject< - TDefinition extends AnyPromptDefinition, - TPromptKind extends PromptKind, - TShape extends SchemaShape, - TKey extends keyof TShape, - TInput, -> = - | BuiltPromptForKind< - ParsedInputOf, - OutputOf, - TPromptKind - > - | PromptInputOverrides, Exclude> - | (undefined extends TInput ? undefined : never); - type BuiltPromptOptions = { definition: { model?: string; @@ -784,11 +636,21 @@ type Extendable = T & { ): Extendable>; }; -type PromptAdapter = ( - builtPrompt: PromptAdapterInput, -) => MaybePromise; +type SyncPromptAdapter = ( + builtPrompt: PromptAdapterInput, +) => TResult; + +type AsyncPromptAdapter< + TInput, + TOutput, + TResult extends PromptAdapterResult, +> = (builtPrompt: PromptAdapterInput) => Promise; + +type PromptAdapter = + | SyncPromptAdapter + | AsyncPromptAdapter; -type PromptAdapterToResult = +type MaybeAsyncPromptAdapterResult = | Extendable | Promise>; @@ -820,8 +682,14 @@ class BuiltMessagesPrompt implements Iterable { } to( - adapter: PromptAdapter, - ): PromptAdapterToResult { + adapter: AsyncPromptAdapter, + ): Promise>; + to( + adapter: SyncPromptAdapter, + ): Extendable; + to( + adapter: PromptAdapter, + ): MaybeAsyncPromptAdapterResult { const result = adapter({ kind: "messages", model: this.definition.model, @@ -857,8 +725,14 @@ class BuiltStringPrompt { } to( - adapter: PromptAdapter, - ): PromptAdapterToResult { + adapter: AsyncPromptAdapter, + ): Promise>; + to( + adapter: SyncPromptAdapter, + ): Extendable; + to( + adapter: PromptAdapter, + ): MaybeAsyncPromptAdapterResult { const result = adapter({ kind: "string", model: this.definition.model, @@ -1242,7 +1116,11 @@ function createPromptVariables( if (templateInfo?.type === "promptDefinition") { return mode === "runtime" ? runtimeValue - : nestedPromptBuilder(templateInfo.definition, templateInfo.kind, path); + : nestedPromptBuilder( + templateInfo.definition as AnyPromptDefinition, + templateInfo.kind, + path, + ); } if (templateInfo?.type === "attachment") { @@ -1424,7 +1302,9 @@ function createTemplateDependencyInput( templateInfo.kind === "messages" ? "template_messages_prompt" : "template_string_prompt", - root: promptDefinitionRoot(templateInfo.definition), + root: promptDefinitionRoot( + templateInfo.definition as AnyPromptDefinition, + ), }; } @@ -1856,211 +1736,6 @@ function buildAnyPrompt( return definition.build(input as never) as AnyBuiltPrompt; } -function stringSchema(): PromptSchema< - string, - string, - unknown, - TDomain -> { - return new PromptSchema( - (value, path) => { - if (typeof value !== "string") { - throw new Error(`${path} must be a string`); - } - return value; - }, - () => ({ type: "string" }), - ); -} - -function numberSchema(): PromptSchema< - number, - number, - unknown, - TDomain -> { - return new PromptSchema( - (value, path) => { - if (typeof value !== "number") { - throw new Error(`${path} must be a number`); - } - return value; - }, - () => ({ type: "number" }), - ); -} - -function booleanSchema(): PromptSchema< - boolean, - boolean, - unknown, - TDomain -> { - return new PromptSchema( - (value, path) => { - if (typeof value !== "boolean") { - throw new Error(`${path} must be a boolean`); - } - return value; - }, - () => ({ type: "boolean" }), - ); -} - -function enumSchema< - const TValues extends readonly [string, ...string[]], - TDomain extends SchemaDomain = "input", ->( - values: TValues, -): PromptSchema { - return new PromptSchema( - (value, path) => { - if (typeof value !== "string" || !values.includes(value)) { - throw new Error(`${path} must be one of ${values.join(", ")}`); - } - return value; - }, - () => ({ type: "string", enum: [...values] }), - ); -} - -function createArraySchema< - TItemSchema extends AnySchema, - TDomain extends SchemaDomain, ->( - item: TItemSchema, -): PromptSchema< - InferSchema[], - InferInputSchema[], - unknown, - TDomain -> { - return new PromptSchema< - InferSchema[], - InferInputSchema[], - unknown, - TDomain - >( - (value, path, root) => { - if (!Array.isArray(value)) { - throw new Error(`${path} must be an array`); - } - return value.map((itemValue, index) => - item.parse(itemValue, `${path}[${index}]`, root), - ) as InferSchema[]; - }, - () => ({ type: "array", items: item.toJSONSchema() }), - false, - { type: "array", item }, - ); -} - -function arraySchema( - item: TItemSchema, -): PromptSchema< - InferSchema[], - InferInputSchema[], - unknown, - "input" -> { - return createArraySchema(item); -} - -function outputArraySchema( - item: TItemSchema, -): PromptSchema< - InferSchema[], - InferInputSchema[], - unknown, - "output" -> { - return createArraySchema(item); -} - -function createObjectSchema< - TShape extends SchemaShape, - TDomain extends SchemaDomain, ->( - shape: TShape, -): PromptSchema< - InferParsedObject, - InferInputObject, - unknown, - TDomain -> { - return new PromptSchema< - InferParsedObject, - InferInputObject, - unknown, - TDomain - >( - (value, path, root) => { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - throw new Error(`${path} must be an object`); - } - const record = value as Record; - const rootInput = root === undefined ? record : root; - return Object.fromEntries( - Object.entries(shape) - .filter(([key, schema]) => key in record || !schema.isOptional) - .map(([key, schema]) => [ - key, - schema.parse(record[key], `${path}.${key}`, rootInput), - ]), - ) as InferParsedObject; - }, - () => ({ - type: "object", - properties: Object.fromEntries( - Object.entries(shape).map(([key, schema]) => [ - key, - schema.toJSONSchema(), - ]), - ), - required: Object.entries(shape) - .filter(([, schema]) => !schema.isOptional) - .map(([key]) => key), - additionalProperties: false, - }), - false, - { type: "object", shape }, - ); -} - -function objectSchema( - shape: TShape, -): PromptSchema< - InferParsedObject, - InferInputObject, - unknown, - "input" -> { - return createObjectSchema(shape); -} - -function outputObjectSchema( - shape: TShape, -): PromptSchema< - InferParsedObject, - InferInputObject, - unknown, - "output" -> { - return createObjectSchema(shape); -} - -function unknownSchema(): PromptSchema< - unknown, - unknown, - unknown, - TDomain -> { - return new PromptSchema( - (value) => value, - () => ({}), - ); -} - function attachmentSchema(): PromptSchema< PromptAttachment, PromptAttachment, @@ -2207,85 +1882,6 @@ function promptDefinitionSchema< ); } -/** - * @internal Converts experimental prompt template data into the existing prompt - * definition shape. This is intended for future backend-saving code paths. - */ -export function promptDefinitionToMustache( - data: ExperimentalPromptData, -): MustachePromptDefinition { - if (!data.model) { - throw new Error("Cannot convert prompt data to mustache without a model"); - } - - if (data.kind === "messages") { - return { - model: data.model, - messages: data.messages.map(promptMessageToMustacheMessage), - }; - } - - return { - model: data.model, - messages: [{ role: "user", content: data.content }], - }; -} - -function promptMessageToMustacheMessage( - message: PromptMessage, -): ChatCompletionMessageParamType { - if (typeof message.content === "string") { - if (message.role === "system") { - return { role: "system", content: message.content }; - } - if (message.role === "assistant") { - return { role: "assistant", content: message.content }; - } - return { role: "user", content: message.content }; - } - if (message.role !== "user") { - throw new Error("Only user messages can contain prompt.file parts"); - } - return { - role: "user", - content: message.content.map(promptContentPartToMustachePart), - }; -} - -function promptContentPartToMustachePart( - part: PromptMessageContentPart, -): ChatCompletionContentPartType { - if (part.type === "text") { - return part; - } - - const value = stringifyTemplateValue(part.file.value).content; - const contentType = - part.file.contentType ?? - (typeof value === "string" ? dataUrlContentType(value) : undefined); - if (isImageContentType(contentType)) { - return { - type: "image_url" as const, - image_url: { - url: value, - ...(part.file.detail ? { detail: part.file.detail } : undefined), - }, - }; - } - - return { - type: "file" as const, - file: { - file_data: value, - ...(part.file.filename ? { filename: part.file.filename } : undefined), - }, - }; -} - -function isImageContentType(contentType: string | undefined): boolean { - return contentType?.startsWith("image/") ?? false; -} - const inputSchemaHelpers = { string: stringSchema, number: numberSchema, diff --git a/js/src/experimental-prompt-api.typecheck.ts b/js/src/experimental-prompt-api.typecheck.ts new file mode 100644 index 000000000..9683cdeb3 --- /dev/null +++ b/js/src/experimental-prompt-api.typecheck.ts @@ -0,0 +1,303 @@ +import { prompt } from "./experimental-prompt-api"; +import type { + PromptAttachment, + PromptMessage, +} from "./experimental-prompt-api"; + +type Extends = T extends U ? true : false; +type Equal = + (() => V extends T ? 1 : 2) extends () => V extends U ? 1 : 2 + ? true + : false; +type Expect = T; + +const classifierPrompt = prompt.define({ + slug: "typecheck-classifier", + model: "gpt-4o-mini", + inputSchema: (s) => + s.object({ + text: s.string(), + count: s.number().optional(), + mode: s.enum(["brief", "full"]), + files: s.array(s.attachment()).optional(), + metadata: s.object({ + urgent: s.boolean(), + }), + }), + outputSchema: (s) => + s.object({ + label: s.enum(["bug", "question"]), + reasons: s.array(s.string()).optional(), + }), + template: ({ variables }) => { + type _TemplateTextIsOpaque = Expect>; + type _NestedTemplateFieldIsVisible = Expect< + Equal + >; + type _ArrayListTagIsCallable = Expect< + Extends< + typeof variables.files.list, + ( + strings: TemplateStringsArray, + ...values: readonly unknown[] + ) => unknown + > + >; + + return [ + prompt.user`Classify ${variables.text} as ${variables.mode}.`, + prompt.user`Urgent: ${variables.metadata.urgent}`, + prompt.user`Files: ${variables.files.list`- ${variables.files.list}\n`}`, + ]; + }, +}); + +type ClassifierInput = Parameters[0]; +type ExpectedClassifierInput = { + text: string; + count?: number; + mode: "brief" | "full"; + files?: PromptAttachment[]; + metadata: { + urgent: boolean; + }; +}; +type _ClassifierInputMatchesExpected = Expect< + Extends +>; +type _ExpectedMatchesClassifierInput = Expect< + Extends +>; + +const classifierInput: ClassifierInput = { + text: "The app crashes", + mode: "brief", + metadata: { urgent: true }, +}; + +classifierPrompt.build(classifierInput); + +classifierPrompt.build({ + text: "The app crashes", + mode: "full", + files: ["data:text/plain;base64,aGVsbG8="], + metadata: { urgent: false }, +}); + +classifierPrompt.build({ + text: "The app crashes", + // @ts-expect-error enum inputs preserve their literal value set + mode: "medium", + metadata: { urgent: true }, +}); + +classifierPrompt.build({ + text: "The app crashes", + mode: "brief", + // @ts-expect-error required nested object fields are enforced + metadata: {}, +}); + +classifierPrompt.build({ + text: "The app crashes", + mode: "brief", + metadata: { urgent: true }, + // @ts-expect-error object schemas reject unknown input keys at compile time + extra: true, +}); + +const classifierBuilt = classifierPrompt.build(classifierInput); +type _ClassifierKindIsMessages = Expect< + Equal +>; +type _ClassifierMessagesArePromptMessages = Expect< + Equal +>; +type ClassifierParsedOutput = ReturnType< + NonNullable["parse"] +>; +type _ClassifierOutputMatchesExpected = Expect< + Extends< + ClassifierParsedOutput, + { + label: "bug" | "question"; + reasons?: string[]; + } + > +>; + +const customAdapterResult = classifierBuilt.to((snapshot) => { + type _SnapshotInputMatchesPromptInput = Expect< + Extends + >; + type _SnapshotOutputMatchesPromptOutput = Expect< + Extends< + ReturnType["parse"]>, + { + label: "bug" | "question"; + reasons?: string[]; + } + > + >; + + if (snapshot.kind === "messages") { + type _SnapshotMessagesArePromptMessages = Expect< + Equal + >; + // @ts-expect-error message prompt adapter snapshots do not expose content + void snapshot.content; + } + + return { + input: snapshot.input, + parsedOutput: snapshot.outputSchema?.parse({ label: "bug" }), + }; +}); +type _CustomAdapterInputPreservesType = Expect< + Extends +>; +type _CustomAdapterOutputPreservesType = Expect< + Extends< + NonNullable, + { + label: "bug" | "question"; + reasons?: string[]; + } + > +>; +const _customAdapterExtended = customAdapterResult.extend({ + nested: { ok: true }, +}); +type _CustomAdapterExtendDeepMerges = Expect< + Equal +>; + +const _asyncCustomAdapterResult = classifierBuilt.to(async (snapshot) => ({ + input: snapshot.input, +})); +type _AsyncCustomAdapterReturnsPromise = Expect< + Extends< + typeof _asyncCustomAdapterResult, + Promise<{ input: ExpectedClassifierInput }> + > +>; + +void (async () => { + const openAIArgs = await classifierBuilt.to(prompt.adapters.openAIChat()); + const extended = openAIArgs.extend({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, + }, + }); + type _OpenAIChatExtendPreservesTemperature = Expect< + Equal + >; + type _OpenAIChatExtendPreservesCaller = Expect< + Equal + >; + type _OpenAIChatExtendPreservesPromptSlug = Expect< + Equal + >; + + // @ts-expect-error extend only accepts objects + void extended.extend("nope"); +})(); + +const stringPrompt = prompt.define({ + slug: "typecheck-string", + inputSchema: (s) => + s.object({ + text: s.string(), + }), + template: ({ variables }) => prompt.text`Summarize ${variables.text}`, +}); + +const stringBuilt = stringPrompt.build({ text: "hello" }); +type _StringKindIsString = Expect>; +type _StringContentIsString = Expect>; +// @ts-expect-error string prompts are not iterable message prompts +void [...stringBuilt]; + +const brandVoicePrompt = prompt.define({ + slug: "typecheck-brand-voice", + inputSchema: (s) => + s.object({ + company: s.string(), + tone: s.string(), + }), + template: ({ variables }) => [ + prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, + ], +}); + +const replyPrompt = prompt.define({ + slug: "typecheck-reply", + inputSchema: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + voice: s.messagesPromptDefinition(brandVoicePrompt), + }), + template: ({ variables }) => [ + ...variables.voice, + prompt.user`Reply to ${variables.ticket}`, + ], +}); + +type ReplyInput = Parameters[0]; +const replyInputWithInheritedCompany: ReplyInput = { + company: "Braintrust", + ticket: "Where is my eval?", + voice: { tone: "direct" }, +}; +const replyInputWithOverriddenCompany: ReplyInput = { + company: "Braintrust", + ticket: "Where is my eval?", + voice: { company: "Acme", tone: "direct" }, +}; +const replyInputWithBuiltPrompt: ReplyInput = { + company: "Braintrust", + ticket: "Where is my eval?", + voice: brandVoicePrompt.build({ company: "Braintrust", tone: "direct" }), +}; +replyPrompt.build(replyInputWithInheritedCompany); +replyPrompt.build(replyInputWithOverriddenCompany); +replyPrompt.build(replyInputWithBuiltPrompt); + +const replyInputMissingNestedField: ReplyInput = { + company: "Braintrust", + ticket: "Where is my eval?", + // @ts-expect-error nested prompt input still needs fields absent from the parent + voice: {}, +}; +void replyInputMissingNestedField; + +const replyInputWithWrongBuiltPromptKind: ReplyInput = { + company: "Braintrust", + ticket: "Where is my eval?", + // @ts-expect-error string prompts cannot satisfy message prompt fields + voice: stringBuilt, +}; +void replyInputWithWrongBuiltPromptKind; + +prompt.define({ + slug: "typecheck-output-helper-scope", + inputSchema: (s) => s.object({ text: s.string() }), + outputSchema: (s) => + s.object({ + ok: s.boolean(), + // @ts-expect-error output schema helpers do not expose prompt helpers + prompt: s.builtMessagesPrompt(), + }), + template: ({ variables }) => [prompt.user`${variables.text}`], +}); + +prompt.define({ + slug: "typecheck-invalid-template-return", + inputSchema: (s) => s.object({ text: s.string() }), + // @ts-expect-error templates must return messages or prompt.text + template: ({ variables }) => prompt.user`${variables.text}`, +}); diff --git a/js/src/template-generators/mustache.ts b/js/src/template-generators/mustache.ts new file mode 100644 index 000000000..ebc617dcd --- /dev/null +++ b/js/src/template-generators/mustache.ts @@ -0,0 +1,117 @@ +import type { + ChatCompletionContentPartType, + ChatCompletionMessageParamType, +} from "../generated_types"; +import type { + ExperimentalPromptData, + PromptMessage, + PromptMessageContentPart, +} from "../experimental-prompt-api"; +import type { PromptDefinition as MustachePromptDefinition } from "../prompt-schemas"; + +/** + * @internal Converts experimental prompt template data into the existing prompt + * definition shape. This is intended for future backend-saving code paths. + */ +export function promptDefinitionToMustache( + data: ExperimentalPromptData, +): MustachePromptDefinition { + if (!data.model) { + throw new Error("Cannot convert prompt data to mustache without a model"); + } + + if (data.kind === "messages") { + return { + model: data.model, + messages: data.messages.map(promptMessageToMustacheMessage), + }; + } + + return { + model: data.model, + messages: [{ role: "user", content: data.content }], + }; +} + +function promptMessageToMustacheMessage( + message: PromptMessage, +): ChatCompletionMessageParamType { + if (typeof message.content === "string") { + if (message.role === "system") { + return { role: "system", content: message.content }; + } + if (message.role === "assistant") { + return { role: "assistant", content: message.content }; + } + return { role: "user", content: message.content }; + } + if (message.role !== "user") { + throw new Error("Only user messages can contain prompt.file parts"); + } + return { + role: "user", + content: message.content.map(promptContentPartToMustachePart), + }; +} + +function promptContentPartToMustachePart( + part: PromptMessageContentPart, +): ChatCompletionContentPartType { + if (part.type === "text") { + return part; + } + + const value = stringifyTemplateValue(part.file.value); + const contentType = + part.file.contentType ?? + (typeof value === "string" ? dataUrlContentType(value) : undefined); + if (isImageContentType(contentType)) { + return { + type: "image_url" as const, + image_url: { + url: value, + ...(part.file.detail ? { detail: part.file.detail } : undefined), + }, + }; + } + + return { + type: "file" as const, + file: { + file_data: value, + ...(part.file.filename ? { filename: part.file.filename } : undefined), + }, + }; +} + +function stringifyTemplateValue(value: unknown): string { + if (value === undefined || value === null) { + return ""; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return String(value); + } + if ( + typeof value === "object" && + value !== null && + (typeof (value as { [Symbol.toPrimitive]?: unknown })[ + Symbol.toPrimitive + ] === "function" || + Object.prototype.hasOwnProperty.call(value, "toString")) + ) { + return String(value); + } + return JSON.stringify(value) ?? ""; +} + +function dataUrlContentType(value: string): string | undefined { + return value.match(/^data:([^;,]+)[;,]/)?.[1]; +} + +function isImageContentType(contentType: string | undefined): boolean { + return contentType?.startsWith("image/") ?? false; +}