diff --git a/js/src/experimental-prompt-adapters.ts b/js/src/experimental-prompt-adapters.ts new file mode 100644 index 000000000..5bd31ca5d --- /dev/null +++ b/js/src/experimental-prompt-adapters.ts @@ -0,0 +1,428 @@ +import { BaseAttachment, ReadonlyAttachment } from "./logger"; +import type { BraintrustState } from "./logger"; +import type { AttachmentReferenceType as AttachmentReference } from "./generated_types"; +import type { + InlineAttachmentReference, + PromptDependencies, + PromptJsonSchema, + PromptMessage, + PromptMessageContentPart, +} from "./experimental-prompt-api"; + +type PromptAdapterInput = { + kind: "messages" | "string"; + model?: string; + inputSchema: { toJSONSchema(): PromptJsonSchema }; + outputSchema?: { toJSONSchema(): PromptJsonSchema }; + input: unknown; + messages: PromptMessage[]; + content?: string; + dependencies: PromptDependencies; +}; + +type OpenAIContentPart = + | { type: "text"; text: string } + | { + type: "image_url"; + image_url: { url: string; detail?: "auto" | "low" | "high" }; + } + | { + type: "file"; + file: { file_data?: string; file_id?: string; filename?: string }; + }; + +type OpenAIChatMessage = Omit & { + content: string | OpenAIContentPart[]; +}; + +type OpenAIChatPromptArgs = { + model?: string; + messages: OpenAIChatMessage[]; + response_format?: { + type: "json_schema"; + json_schema: { + name: string; + schema: PromptJsonSchema; + strict: true; + }; + }; + span_info: { + metadata: { + prompt: PromptDependencies; + }; + }; +}; + +type AISDKContentPart = + | { type: "text"; text: string } + | { type: "image"; image: string; mediaType?: string } + | { type: "file"; data: string; mediaType: string; filename?: string }; + +type AISDKMessage = Omit & { + content: string | AISDKContentPart[]; +}; + +type AISDKGenerateObjectPromptArgs = { + model?: string; + messages: AISDKMessage[]; + schema?: PromptJsonSchema; + experimental_telemetry: { + metadata: { + braintrustPrompt: PromptDependencies; + }; + }; +}; + +type AdapterOptions = { + state?: BraintrustState; +}; + +type OpenAIChatAdapterOptions = AdapterOptions; +type AISDKGenerateObjectAdapterOptions = AdapterOptions; + +type ResolvedPromptFile = { + data: string; + contentType?: string; + filename?: string; + detail?: "auto" | "low" | "high"; +}; + +function schemaName(slug: string): string { + const name = slug.replace(/[^a-zA-Z0-9_-]/g, "_"); + return name.length > 0 ? `${name}_output` : "prompt_output"; +} + +function openAIChatAdapter( + options: OpenAIChatAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => { + const outputSchema = builtPrompt.outputSchema?.toJSONSchema(); + return { + model: builtPrompt.model, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderOpenAIMessage(message, options), + ), + ), + ...(outputSchema + ? { + response_format: { + type: "json_schema" as const, + json_schema: { + name: schemaName(builtPrompt.dependencies.root.slug), + schema: outputSchema, + strict: true as const, + }, + }, + } + : undefined), + span_info: { + metadata: { + prompt: builtPrompt.dependencies, + }, + }, + }; + }; +} + +async function renderOpenAIMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as OpenAIChatMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderOpenAIContentPart(part, options)), + ), + }; +} + +async function renderOpenAIContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image_url", + image_url: { + url: resolved.data, + ...(resolved.detail ? { detail: resolved.detail } : undefined), + }, + }; + } + + return { + type: "file", + file: isLikelyFileId(resolved.data) + ? { + file_id: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + } + : { + file_data: resolved.data, + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }, + }; +} + +function aiSDKGenerateObjectAdapter( + options: AISDKGenerateObjectAdapterOptions = {}, +): (builtPrompt: PromptAdapterInput) => Promise { + return async (builtPrompt) => ({ + model: builtPrompt.model, + messages: await Promise.all( + builtPrompt.messages.map((message) => + renderAISDKMessage(message, options), + ), + ), + schema: builtPrompt.outputSchema?.toJSONSchema(), + experimental_telemetry: { + metadata: { + braintrustPrompt: builtPrompt.dependencies, + }, + }, + }); +} + +async function renderAISDKMessage( + message: PromptMessage, + options: AdapterOptions, +): Promise { + if (typeof message.content === "string") { + return message as AISDKMessage; + } + return { + ...message, + content: await Promise.all( + message.content.map((part) => renderAISDKContentPart(part, options)), + ), + }; +} + +async function renderAISDKContentPart( + part: PromptMessageContentPart, + options: AdapterOptions, +): Promise { + if (part.type === "text") { + return part; + } + + const resolved = await resolvePromptFile(part, options); + if (isImageFile(resolved)) { + return { + type: "image", + image: resolved.data, + ...(resolved.contentType + ? { mediaType: resolved.contentType } + : undefined), + }; + } + + return { + type: "file", + data: resolved.data, + mediaType: resolved.contentType ?? "application/octet-stream", + ...(resolved.filename ? { filename: resolved.filename } : undefined), + }; +} + +async function resolvePromptFile( + part: Extract, + options: AdapterOptions, +): Promise { + const value = part.file.value; + const optionContentType = part.file.contentType; + const optionFilename = part.file.filename; + + if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { + const reference = value.reference; + const data = + value instanceof ReadonlyAttachment + ? await value.asBase64Url() + : await blobToDataUrl(await value.data(), reference.content_type); + return { + data, + contentType: optionContentType ?? reference.content_type, + filename: optionFilename ?? reference.filename, + detail: part.file.detail, + }; + } + + if (isAttachmentReference(value)) { + const data = await new ReadonlyAttachment( + value, + options.state, + ).asBase64Url(); + return { + data, + contentType: optionContentType ?? value.content_type, + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isInlineAttachmentReference(value)) { + const data = value.data ?? value.src; + return { + data, + contentType: + optionContentType ?? value.content_type ?? contentTypeFromString(data), + filename: optionFilename ?? value.filename, + detail: part.file.detail, + }; + } + + if (isBlob(value)) { + const contentType = optionContentType ?? (value.type || undefined); + return { + data: await blobToDataUrl(value, contentType), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) { + const contentType = optionContentType ?? "application/octet-stream"; + return { + data: await blobToDataUrl( + new Blob([value as BlobPart], { type: contentType }), + contentType, + ), + contentType, + filename: optionFilename, + detail: part.file.detail, + }; + } + + if (typeof value === "string") { + return { + data: value, + contentType: optionContentType ?? contentTypeFromString(value), + filename: optionFilename, + detail: part.file.detail, + }; + } + + throw new Error("prompt.file value must be an attachment-compatible value"); +} + +function isImageFile(file: ResolvedPromptFile): boolean { + if (file.contentType?.startsWith("image/")) { + return true; + } + if (file.contentType && !file.contentType.startsWith("image/")) { + return false; + } + return isHttpUrl(file.data); +} + +function isLikelyFileId(value: string): boolean { + return !isHttpUrl(value) && !value.startsWith("data:"); +} + +function contentTypeFromString(value: string): string | undefined { + const dataUrlContentType = value.match(/^data:([^;,]+)[;,]/)?.[1]; + if (dataUrlContentType) { + return dataUrlContentType; + } + + const extension = value.split(/[?#]/, 1)[0]?.split(".").at(-1)?.toLowerCase(); + switch (extension) { + case "png": + return "image/png"; + case "jpg": + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "pdf": + return "application/pdf"; + case "txt": + return "text/plain"; + case "json": + return "application/json"; + default: + return undefined; + } +} + +function isHttpUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== "undefined" && value instanceof Blob; +} + +function isAttachmentReference(value: unknown): value is AttachmentReference { + return ( + isRecord(value) && + ((value.type === "braintrust_attachment" && + typeof value.key === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string") || + (value.type === "external_attachment" && + typeof value.url === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string")) + ); +} + +function isInlineAttachmentReference( + value: unknown, +): value is InlineAttachmentReference { + return ( + isRecord(value) && + value.type === "inline_attachment" && + typeof value.src === "string" && + (value.content_type === undefined || + typeof value.content_type === "string") && + (value.filename === undefined || typeof value.filename === "string") && + (value.data === undefined || typeof value.data === "string") + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function blobToDataUrl( + blob: Blob, + contentType?: string, +): Promise { + const buffer = await blob.arrayBuffer(); + const base64 = + typeof Buffer !== "undefined" + ? Buffer.from(buffer).toString("base64") + : bytesToBase64(new Uint8Array(buffer)); + return `data:${contentType ?? (blob.type || "application/octet-stream")};base64,${base64}`; +} + +function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + return btoa(binary); +} + +export const adapters = { + openAIChat: openAIChatAdapter, + aiSDKGenerateObject: aiSDKGenerateObjectAdapter, +}; diff --git a/js/src/experimental-prompt-api.test.ts b/js/src/experimental-prompt-api.test.ts new file mode 100644 index 000000000..8f5795465 --- /dev/null +++ b/js/src/experimental-prompt-api.test.ts @@ -0,0 +1,1141 @@ +import { describe, expect, test } from "vitest"; +import { prompt, promptDefinitionToMustache } from "./experimental-prompt-api"; +import { Attachment, ReadonlyAttachment } from "./logger"; + +describe("experimental prompt API", () => { + test("builds message prompts and translates to OpenAI chat args", async () => { + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + ticket: s.string(), + }), + outputSchema: (s) => + s.object({ + subject: s.string(), + body: s.string(), + urgency: s.enum(["low", "medium", "high"]), + }), + template: ({ variables }) => [ + prompt.system`You write concise support replies.`, + prompt.user`Ticket: ${variables.ticket}`, + ], + }); + + const built = supportReply.build({ + ticket: "I cannot find eval history.", + }); + const output = { + subject: "Finding eval history", + body: "Here is where to look.", + urgency: "low", + }; + + expect(built.kind).toBe("messages"); + expect("content" in built).toBe(false); + expect([...built]).toEqual(built.messages); + expect(built.definition.outputSchema?.parse(output, "output")).toEqual( + output, + ); + await expect(built.to(prompt.adapters.openAIChat())).resolves.toMatchObject( + { + model: "gpt-4o", + messages: [ + { role: "system", content: "You write concise support replies." }, + { role: "user", content: "Ticket: I cannot find eval history." }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "support-reply_output", + strict: true, + schema: { + type: "object", + required: ["subject", "body", "urgency"], + }, + }, + }, + span_info: { + metadata: { + prompt: { + root: { slug: "support-reply" }, + prompts: [ + { + slug: "support-reply", + role: "root", + input: { ticket: "I cannot find eval history." }, + }, + ], + }, + }, + }, + }, + ); + + if (false) { + // @ts-expect-error message prompts do not expose string content + void built.content; + } + }); + + test("builds string prompts and coerces adapters to a user message", async () => { + const policyText = prompt.define({ + slug: "policy-text", + model: "gpt-4o-mini", + inputSchema: (s) => + s.object({ + policy: s.string(), + }), + template: ({ variables }) => prompt.text`Policy: ${variables.policy}`, + }); + + const built = policyText.build({ policy: "Prefer short answers." }); + + expect(built.kind).toBe("string"); + expect(built.content).toBe("Policy: Prefer short answers."); + expect("messages" in built).toBe(false); + await expect(built.to(prompt.adapters.openAIChat())).resolves.toMatchObject( + { + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Policy: Prefer short answers." }], + span_info: { + metadata: { + prompt: { + root: { slug: "policy-text" }, + }, + }, + }, + }, + ); + expect( + built.to((snapshot) => ({ + kind: snapshot.kind, + content: snapshot.kind === "string" ? snapshot.content : undefined, + messages: snapshot.messages, + })), + ).toEqual({ + kind: "string", + content: "Policy: Prefer short answers.", + messages: [{ role: "user", content: "Policy: Prefer short answers." }], + }); + + if (false) { + // @ts-expect-error string prompts do not expose messages + void built.messages; + // @ts-expect-error string prompts are not iterable + void [...built]; + } + }); + + test("exports message prompt data as inlined mustache templates", () => { + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + customer: s.object({ + name: s.string(), + }), + ticket: s.string(), + }), + outputSchema: (s) => + s.object({ + body: s.string(), + }), + template: ({ variables }) => [ + prompt.system`You write concise support replies.`, + prompt.user`Customer: ${variables.customer.name}\nTicket: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data).toMatchObject({ + slug: "support-reply", + model: "gpt-4o", + kind: "messages", + inputSchema: { + type: "object", + required: ["customer", "ticket"], + }, + outputSchema: { + type: "object", + required: ["body"], + }, + messages: [ + { role: "system", content: "You write concise support replies." }, + { + role: "user", + content: "Customer: {{customer.name}}\nTicket: {{ticket}}", + }, + ], + }); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o", + messages: [ + { role: "system", content: "You write concise support replies." }, + { + role: "user", + content: "Customer: {{customer.name}}\nTicket: {{ticket}}", + }, + ], + }); + }); + + test("exports string prompt data as a mustache user message template", () => { + const summarize = prompt.define({ + slug: "summarize", + model: "gpt-4o-mini", + inputSchema: (s) => + s.object({ + text: s.string(), + }), + template: ({ variables }) => prompt.text`Summarize: ${variables.text}`, + }); + + const data = summarize.toPromptData(); + + expect(data).toMatchObject({ + slug: "summarize", + model: "gpt-4o-mini", + kind: "string", + content: "Summarize: {{text}}", + }); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o-mini", + messages: [{ role: "user", content: "Summarize: {{text}}" }], + }); + }); + + test("inlines nested message prompt definitions in mustache prompt data", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + version: "v3", + inputSchema: (s) => + s.object({ + company: s.string(), + tone: s.string(), + }), + template: ({ variables }) => [ + prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, + ], + }); + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + voice: s.messagesPromptDefinition(brandVoice), + }), + template: ({ variables }) => [ + ...variables.voice, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "system", + content: "Use {{company}}'s {{voice.tone}} voice.", + }, + { role: "user", content: "Draft a reply for: {{ticket}}" }, + ]); + expect(data.dependencies.prompts).toMatchObject([ + { slug: "support-reply", role: "root" }, + { + slug: "brand-voice", + version: "v3", + role: "include", + parent: "support-reply", + }, + ]); + }); + + test("inlines nested string prompt definitions in mustache prompt data", () => { + const policyText = prompt.define({ + slug: "policy-text", + version: "v2", + inputSchema: (s) => + s.object({ + company: s.string(), + text: s.string(), + }), + template: ({ variables }) => + prompt.text`${variables.company}: ${variables.text}`, + }); + const supportReply = prompt.define({ + slug: "support-reply", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + policy: s.stringPromptDefinition(policyText), + }), + template: ({ variables }) => [ + prompt.system`Follow this policy: ${variables.policy}`, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + const data = supportReply.toPromptData(); + + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "system", + content: "Follow this policy: {{company}}: {{policy.text}}", + }, + { role: "user", content: "Draft a reply for: {{ticket}}" }, + ]); + expect(data.dependencies.prompts).toMatchObject([ + { slug: "support-reply", role: "root" }, + { + slug: "policy-text", + version: "v2", + role: "include", + parent: "support-reply", + }, + ]); + }); + + test("promptDefinitionToMustache requires a model", () => { + const modeless = prompt.define({ + slug: "modeless", + inputSchema: (s) => s.object({ text: s.string() }), + template: ({ variables }) => [prompt.user`Say ${variables.text}`], + }); + + expect(() => promptDefinitionToMustache(modeless.toPromptData())).toThrow( + "Cannot convert prompt data to mustache without a model", + ); + }); + + test("exports array list templates as mustache sections", () => { + const itemList = prompt.define({ + slug: "item-list", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + items: s.array( + s.object({ + foobar: s.string(), + author: s.object({ + name: s.string(), + }), + }), + ), + }), + template: ({ variables }) => [ + prompt.user`Items:\n${variables.items.list`- ${variables.items.list.foobar} by ${variables.items.list.author.name}\n`}`, + ], + }); + + const built = itemList.build({ + items: [ + { foobar: "first", author: { name: "Ada" } }, + { foobar: "second", author: { name: "Grace" } }, + ], + }); + const data = itemList.toPromptData(); + + expect(built.messages).toEqual([ + { + role: "user", + content: "Items:\n- first by Ada\n- second by Grace\n", + }, + ]); + expect(data.kind).toBe("messages"); + expect(data.messages).toEqual([ + { + role: "user", + content: + "Items:\n{{#items}}- {{foobar}} by {{author.name}}\n{{/items}}", + }, + ]); + expect(promptDefinitionToMustache(data)).toEqual({ + model: "gpt-4o", + messages: [ + { + role: "user", + content: + "Items:\n{{#items}}- {{foobar}} by {{author.name}}\n{{/items}}", + }, + ], + }); + }); + + test("template contexts do not expose parsed runtime values", () => { + if (false) { + prompt.define({ + slug: "no-runtime-values", + inputSchema: (s) => + s.object({ + items: s.array( + s.object({ + foobar: s.string(), + }), + ), + }), + // @ts-expect-error template contexts only expose template variables + template: ({ values }) => [ + prompt.user`Second: ${values.items[1]?.foobar}`, + ], + }); + } + }); + + test("converts prompt.file parts for OpenAI and AI SDK adapters", async () => { + const imageDataUrl = "data:image/png;base64,aW1hZ2U="; + const pdfDataUrl = "data:application/pdf;base64,cGRm"; + const describeFiles = prompt.define({ + slug: "describe-files", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + image: s.attachment(), + document: s.attachment(), + secondImage: s.attachment(), + }), + template: ({ variables }) => [ + prompt.user([ + prompt.text`Describe these files.`, + prompt.file(variables.image), + prompt.file(variables.document, { filename: "brief.pdf" }), + prompt.file(variables.secondImage), + ]), + ], + }); + + const built = describeFiles.build({ + image: imageDataUrl, + document: pdfDataUrl, + secondImage: "https://example.com/second.jpg", + }); + const openAIArgs = await built.to(prompt.adapters.openAIChat()); + const aiSDKArgs = await built.to(prompt.adapters.aiSDKGenerateObject()); + + expect(openAIArgs.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these files." }, + { type: "image_url", image_url: { url: imageDataUrl } }, + { + type: "file", + file: { file_data: pdfDataUrl, filename: "brief.pdf" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/second.jpg" }, + }, + ]); + expect(aiSDKArgs.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these files." }, + { type: "image", image: imageDataUrl, mediaType: "image/png" }, + { + type: "file", + data: pdfDataUrl, + mediaType: "application/pdf", + filename: "brief.pdf", + }, + { + type: "image", + image: "https://example.com/second.jpg", + mediaType: "image/jpeg", + }, + ]); + }); + + test("resolves Attachment and ReadonlyAttachment values without leaking bytes", async () => { + const attachment = new Attachment({ + data: new Blob(["hello"], { type: "text/plain" }), + filename: "hello.txt", + contentType: "text/plain", + }); + const readonly = new ReadonlyAttachment({ + type: "external_attachment", + url: "https://example.com/doc.pdf", + filename: "doc.pdf", + content_type: "application/pdf", + }); + readonly.asBase64Url = async () => "data:application/pdf;base64,cGRm"; + const describeFiles = prompt.define({ + slug: "describe-uploaded-files", + model: "gpt-4o", + inputSchema: (s) => + s.object({ + attachment: s.attachment(), + readonly: s.attachment(), + }), + template: ({ variables }) => [ + prompt.user([ + prompt.text`Describe these uploads.`, + prompt.file(variables.attachment), + prompt.file(variables.readonly), + ]), + ], + }); + + const built = describeFiles.build({ attachment, readonly }); + const args = await built.to(prompt.adapters.openAIChat()); + + expect(args.messages[0]?.content).toEqual([ + { type: "text", text: "Describe these uploads." }, + { + type: "file", + file: { + file_data: "data:text/plain;base64,aGVsbG8=", + filename: "hello.txt", + }, + }, + { + type: "file", + file: { + file_data: "data:application/pdf;base64,cGRm", + filename: "doc.pdf", + }, + }, + ]); + expect(JSON.stringify(built.dependencies.prompts[0]?.input)).not.toContain( + "aGVsbG8=", + ); + expect(built.dependencies.prompts[0]?.input).toMatchObject({ + attachment: { + type: "attachment", + reference: { + type: "braintrust_attachment", + filename: "hello.txt", + content_type: "text/plain", + }, + }, + readonly: { + type: "attachment", + reference: { + type: "external_attachment", + filename: "doc.pdf", + content_type: "application/pdf", + }, + }, + }); + }); + + test("rejects rich media content outside user messages", () => { + const invalid = prompt.define({ + slug: "invalid-media-role", + inputSchema: (s) => s.object({}), + template: () => [ + { + role: "system" as const, + content: [prompt.file("https://example.com/image.png")], + }, + ], + }); + + expect(() => invalid.build({})).toThrow( + "template[0] must be a prompt message", + ); + }); + + test("extends adapter args with a typed deep merge", async () => { + const classify = prompt.define({ + slug: "classify", + model: "gpt-4o-mini", + inputSchema: (s) => + s.object({ + text: s.string(), + }), + outputSchema: (s) => + s.object({ + label: s.enum(["bug", "question"]), + }), + template: ({ variables }) => [prompt.user`Classify: ${variables.text}`], + }); + + const built = classify.build({ text: "It crashes" }); + const args = await built.to(prompt.adapters.openAIChat()); + const extended = args.extend({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, + }, + }); + const extendedAgain = extended.extend({ + top_p: 0.5, + span_info: { + metadata: { + requestId: "req_123", + }, + }, + }); + + expect(Object.keys(args)).not.toContain("extend"); + expect({ ...args }).not.toHaveProperty("extend"); + expect(extended).toMatchObject({ + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + prompt: { + root: { slug: "classify" }, + }, + }, + }, + }); + expect(extendedAgain).toMatchObject({ + temperature: 0.2, + top_p: 0.5, + span_info: { + metadata: { + caller: "support-workflow", + requestId: "req_123", + prompt: { + root: { slug: "classify" }, + }, + }, + }, + }); + expect(() => built.to(() => "nope" as never)).toThrow( + "prompt adapters must return an object", + ); + expect(() => built.to(() => [] as never)).toThrow( + "prompt adapters must return an object", + ); + expect(() => args.extend("nope" as never)).toThrow( + "extend must receive an object", + ); + + if (false) { + // @ts-expect-error adapters must return objects + void built.to(() => "nope"); + + void (async () => { + const typedArgs = (await built.to(prompt.adapters.openAIChat())).extend( + { + temperature: 0.2, + span_info: { + metadata: { + caller: "support-workflow", + }, + }, + }, + ); + const temperature: number = typedArgs.temperature; + const caller: string = typedArgs.span_info.metadata.caller; + const slug: string = typedArgs.span_info.metadata.prompt.root.slug; + void temperature; + void caller; + void slug; + + // @ts-expect-error extend only accepts objects + void typedArgs.extend("nope"); + }); + } + }); + + test("auto-builds message prompt inputs and preserves spread dependencies", () => { + const brandVoice = prompt.define({ + slug: "brand-voice", + version: "v3", + inputSchema: (s) => + s.object({ + company: s.string(), + tone: s.string(), + }), + template: ({ variables }) => [ + prompt.system`Use ${variables.company}'s ${variables.tone} voice.`, + ], + }); + + const supportReply = prompt.define({ + slug: "support-reply", + version: "v8", + inputSchema: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + voice: s.messagesPromptDefinition(brandVoice), + }), + template: ({ variables }) => [ + ...variables.voice, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + type SupportReplyInput = Parameters[0]; + const input: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { tone: "direct" }, + }; + const overriddenNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { company: "Acme", tone: "direct" }, + }; + void overriddenNestedPromptInput; + const missingNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error nested prompt input still needs fields that are not supplied by the parent input + voice: {}, + }; + void missingNestedPromptInput; + const unbuiltNestedPromptInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error raw prompt definitions are not prompt inputs + voice: brandVoice, + }; + void unbuiltNestedPromptInput; + const dynamicPromptDefinitionInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error prompt definition payloads are not prompt inputs + voice: { prompt: brandVoice, input: { tone: "direct" } }, + }; + void dynamicPromptDefinitionInput; + + const built = supportReply.build(input); + + expect(built.messages).toEqual([ + { + role: "system", + content: "Use Braintrust's direct voice.", + }, + { + role: "user", + content: "Draft a reply for: Where did my experiment go?", + }, + ]); + expect(built.dependencies.prompts).toMatchObject([ + { + slug: "support-reply", + role: "root", + input: { + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: { + type: "built_messages_prompt", + root: { slug: "brand-voice", version: "v3" }, + }, + }, + }, + { + slug: "brand-voice", + version: "v3", + role: "include", + parent: "support-reply", + input: { + company: "Braintrust", + tone: "direct", + }, + }, + ]); + expect(() => + supportReply.build({ + company: "Braintrust", + ticket: "Where did my experiment go?", + voice: brandVoice as never, + }), + ).toThrow("input.voice must be a built messages prompt or prompt input"); + }); + + test("auto-builds string prompt inputs and preserves interpolation dependencies", () => { + const policyText = prompt.define({ + slug: "policy-text", + version: "v2", + inputSchema: (s) => + s.object({ + company: s.string(), + policy: s.string(), + }), + template: ({ variables }) => + prompt.text`${variables.company}: ${variables.policy}`, + }); + + const supportReply = prompt.define({ + slug: "support-reply", + inputSchema: (s) => + s.object({ + company: s.string(), + ticket: s.string(), + policy: s.stringPromptDefinition(policyText), + }), + template: ({ variables }) => [ + prompt.system`Follow this policy: ${variables.policy}`, + prompt.user`Draft a reply for: ${variables.ticket}`, + ], + }); + + type SupportReplyInput = Parameters[0]; + const input: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + policy: { policy: "Be concise." }, + }; + const invalidPolicyInput: SupportReplyInput = { + company: "Braintrust", + ticket: "Where did my experiment go?", + // @ts-expect-error nested string prompt input still needs fields that are not supplied by the parent input + policy: {}, + }; + void invalidPolicyInput; + + const built = supportReply.build(input); + + expect(built.messages).toEqual([ + { + role: "system", + content: "Follow this policy: Braintrust: Be concise.", + }, + { + role: "user", + content: "Draft a reply for: Where did my experiment go?", + }, + ]); + expect(built.dependencies.prompts).toMatchObject([ + { + slug: "support-reply", + role: "root", + input: { + company: "Braintrust", + ticket: "Where did my experiment go?", + policy: { + type: "built_string_prompt", + root: { slug: "policy-text", version: "v2" }, + }, + }, + }, + { + slug: "policy-text", + version: "v2", + role: "include", + parent: "support-reply", + input: { + company: "Braintrust", + policy: "Be concise.", + }, + }, + ]); + }); + + test("requires matching built prompt kinds for dynamic schemas", () => { + const messagePrompt = prompt.define({ + slug: "message-prompt", + inputSchema: (s) => + s.object({ + topic: s.string(), + }), + template: ({ variables }) => [ + prompt.user`Message about ${variables.topic}`, + ], + }); + const stringPrompt = prompt.define({ + slug: "string-prompt", + inputSchema: (s) => + s.object({ + topic: s.string(), + }), + template: ({ variables }) => prompt.text`String about ${variables.topic}`, + }); + + const consumeBoth = prompt.define({ + slug: "consume-both", + inputSchema: (s) => + s.object({ + messagePart: s.builtMessagesPrompt(), + stringPart: s.builtStringPrompt(), + }), + template: ({ variables }) => [ + ...variables.messagePart, + prompt.user`Fragment: ${variables.stringPart}`, + ], + }); + + const messagePart = messagePrompt.build({ topic: "tracing" }); + const stringPart = stringPrompt.build({ topic: "evals" }); + + type ConsumeBothInput = Parameters[0]; + const wrongMessageKind: ConsumeBothInput = { + // @ts-expect-error string prompts cannot satisfy message prompt schemas + messagePart: stringPart, + stringPart, + }; + void wrongMessageKind; + const wrongStringKind: ConsumeBothInput = { + messagePart, + // @ts-expect-error message prompts cannot satisfy string prompt schemas + stringPart: messagePart, + }; + void wrongStringKind; + const rawDefinitionInput: ConsumeBothInput = { + // @ts-expect-error raw prompt definitions are not prompt inputs + messagePart: messagePrompt, + stringPart, + }; + void rawDefinitionInput; + const dynamicPayloadInput: ConsumeBothInput = { + messagePart: { + // @ts-expect-error prompt definition payloads are not prompt inputs + prompt: messagePrompt, + input: { topic: "tracing" }, + }, + stringPart, + }; + void dynamicPayloadInput; + + const built = consumeBoth.build({ messagePart, stringPart }); + + expect(built.messages).toEqual([ + { role: "user", content: "Message about tracing" }, + { role: "user", content: "Fragment: String about evals" }, + ]); + expect(() => + consumeBoth.build({ messagePart: stringPart as never, stringPart }), + ).toThrow("input.messagePart must be a built messages prompt"); + expect(() => + consumeBoth.build({ messagePart, stringPart: messagePart as never }), + ).toThrow("input.stringPart must be a built string prompt"); + }); + + test("preserves dependencies through spread and interpolation outside schema inputs", () => { + const messagePrompt = prompt.define({ + slug: "message-prompt", + inputSchema: (s) => + s.object({ + topic: s.string(), + }), + template: ({ variables }) => [ + prompt.user`Message about ${variables.topic}`, + ], + }); + const stringPrompt = prompt.define({ + slug: "string-prompt", + inputSchema: (s) => + s.object({ + topic: s.string(), + }), + template: ({ variables }) => prompt.text`String about ${variables.topic}`, + }); + + const messagePart = messagePrompt.build({ topic: "tracing" }); + const stringPart = stringPrompt.build({ topic: "evals" }); + const wrapper = prompt.define({ + slug: "wrapper", + inputSchema: (s) => s.object({}), + template: () => [...messagePart, prompt.user`Fragment: ${stringPart}`], + }); + + const built = wrapper.build({}); + + expect(built.dependencies.prompts).toMatchObject([ + { slug: "wrapper", role: "root" }, + { slug: "message-prompt", role: "include", parent: "wrapper" }, + { slug: "string-prompt", role: "include", parent: "wrapper" }, + ]); + }); + + test("validates build input, output, and template results", () => { + const typedPrompt = prompt.define({ + slug: "typed", + inputSchema: (s) => + s.object({ + count: s.number(), + }), + outputSchema: (s) => + s.object({ + ok: s.boolean(), + }), + template: ({ variables }) => [prompt.user`Count: ${variables.count}`], + }); + const invalidTemplatePrompt = prompt.define({ + slug: "invalid-template", + inputSchema: (s) => s.object({}), + // @ts-expect-error template must return a message array or prompt.text + template: () => prompt.user`Nope`, + }); + + expect(() => + // @ts-expect-error runtime validation rejects invalid input too + typedPrompt.build({ count: "nope" }), + ).toThrow("input.count must be a number"); + const built = typedPrompt.build({ count: 1 }); + expect(() => + built.definition.outputSchema?.parse({ ok: "yes" }, "output"), + ).toThrow("output.ok must be a boolean"); + expect(() => invalidTemplatePrompt.build({})).toThrow( + "template must return a message array or prompt.text", + ); + }); + + test("passes scoped schema helpers to inputSchema and outputSchema callbacks", () => { + let inputHelperKeys: string[] = []; + let outputHelperKeys: string[] = []; + prompt.define({ + slug: "schema-helper-scopes", + inputSchema: (s) => { + inputHelperKeys = Object.keys(s).sort(); + return s.object({ + topic: s.string(), + }); + }, + outputSchema: (s) => { + outputHelperKeys = Object.keys(s).sort(); + return s.object({ + ok: s.boolean(), + }); + }, + template: ({ variables }) => [prompt.user`Topic: ${variables.topic}`], + }); + + expect(inputHelperKeys).toContain("builtMessagesPrompt"); + expect(inputHelperKeys).toContain("stringPromptDefinition"); + expect(outputHelperKeys).toContain("object"); + expect(outputHelperKeys).not.toContain("builtMessagesPrompt"); + expect(outputHelperKeys).not.toContain("builtStringPrompt"); + expect(outputHelperKeys).not.toContain("messagesPromptDefinition"); + expect(outputHelperKeys).not.toContain("stringPromptDefinition"); + + if (false) { + prompt.define({ + slug: "prompt-output-field", + inputSchema: (s) => s.object({}), + outputSchema: (s) => + s.object({ + // @ts-expect-error output schema helpers do not include prompt helpers + prompt: s.builtMessagesPrompt(), + }), + template: () => [prompt.user`Nope`], + }); + + prompt.define({ + slug: "prompt-output-array", + inputSchema: (s) => s.object({}), + outputSchema: (s) => + // @ts-expect-error output schema helpers do not include prompt helpers + s.array(s.builtStringPrompt()), + template: () => [prompt.user`Nope`], + }); + } + }); + + test("passes flat prompt snapshots to custom adapters", () => { + const classify = prompt.define({ + slug: "classify", + model: "gpt-4o-mini", + inputSchema: (s) => + s.object({ + text: s.string(), + }), + outputSchema: (s) => + s.object({ + label: s.enum(["bug", "question"]), + }), + template: ({ variables }) => [prompt.user`Classify: ${variables.text}`], + }); + const summarize = prompt.define({ + slug: "summarize", + inputSchema: (s) => + s.object({ + text: s.string(), + }), + template: ({ variables }) => prompt.text`Summarize: ${variables.text}`, + }); + + expect( + classify.build({ text: "It crashes" }).to((snapshot) => ({ + keys: Object.keys(snapshot).sort(), + kind: snapshot.kind, + inputSchema: snapshot.inputSchema.toJSONSchema(), + outputSchema: snapshot.outputSchema?.toJSONSchema(), + input: snapshot.input, + })), + ).toMatchObject({ + keys: [ + "dependencies", + "input", + "inputSchema", + "kind", + "messages", + "model", + "outputSchema", + ], + kind: "messages", + inputSchema: { + type: "object", + required: ["text"], + }, + outputSchema: { + type: "object", + required: ["label"], + }, + input: { text: "It crashes" }, + }); + expect( + summarize.build({ text: "It crashes" }).to((snapshot) => ({ + keys: Object.keys(snapshot).sort(), + kind: snapshot.kind, + content: snapshot.kind === "string" ? snapshot.content : undefined, + messages: snapshot.messages, + })), + ).toEqual({ + keys: [ + "content", + "dependencies", + "input", + "inputSchema", + "kind", + "messages", + "model", + "outputSchema", + ], + kind: "string", + content: "Summarize: It crashes", + messages: [{ role: "user", content: "Summarize: It crashes" }], + }); + }); + + test("input schema helper exposes only the explicit built prompt helpers", () => { + prompt.define({ + slug: "input-helper-surface", + inputSchema: (s) => { + expect("builtMessagesPrompt" in s).toBe(true); + expect("builtStringPrompt" in s).toBe(true); + expect("messagesPromptDefinition" in s).toBe(true); + expect("stringPromptDefinition" in s).toBe(true); + expect("attachment" in s).toBe(true); + expect("prompt" in s).toBe(false); + expect("builtPrompt" in s).toBe(false); + expect("promptDefinition" in s).toBe(false); + + if (false) { + // @ts-expect-error built message prompt schemas only accept already-built prompts + void s.builtMessagesPrompt(undefined); + // @ts-expect-error built string prompt schemas only accept already-built prompts + void s.builtStringPrompt(undefined); + // @ts-expect-error message prompt definition schemas require a definition + void s.messagesPromptDefinition(); + // @ts-expect-error string prompt definition schemas require a definition + void s.stringPromptDefinition(); + // @ts-expect-error old generic prompt schema helper was removed + void s.prompt; + // @ts-expect-error old generic built prompt schema helper was removed + void s.builtPrompt; + // @ts-expect-error prompt definitions are no longer accepted as schema inputs + void s.promptDefinition; + // @ts-expect-error rich content is only supported by prompt.user + void prompt.system([prompt.file("https://example.com/image.png")]); + } + + return s.object({}); + }, + template: () => [prompt.user`Ok`], + }); + }); +}); diff --git a/js/src/experimental-prompt-api.ts b/js/src/experimental-prompt-api.ts new file mode 100644 index 000000000..729cf1ac6 --- /dev/null +++ b/js/src/experimental-prompt-api.ts @@ -0,0 +1,2326 @@ +import { adapters } from "./experimental-prompt-adapters"; +import { BaseAttachment, ReadonlyAttachment } from "./logger"; +import type { + AttachmentReferenceType as AttachmentReference, + ChatCompletionContentPartType, + ChatCompletionMessageParamType, +} from "./generated_types"; +import type { PromptDefinition as MustachePromptDefinition } from "./prompt-schemas"; + +type JsonPrimitive = string | number | boolean | null; + +export type PromptJsonSchema = { + type?: string; + properties?: Record; + required?: string[]; + items?: PromptJsonSchema; + enum?: JsonPrimitive[]; + additionalProperties?: boolean; + description?: string; + "x-bt-type"?: string; +}; + +type SchemaParser = (value: unknown, path: string, root: unknown) => T; + +type SchemaDomain = "input" | "output"; +type PromptKind = "messages" | "string"; + +type PromptSchemaTemplateInfo = + | { + type: "object"; + shape: SchemaShape; + } + | { + type: "array"; + item: AnySchema; + } + | { + type: "promptDefinition"; + definition: AnyPromptDefinition; + kind: PromptKind; + } + | { + type: "attachment"; + }; + +class PromptSchema< + TParsed, + TInput = TParsed, + TKind = unknown, + TDomain extends SchemaDomain = "input", +> { + readonly _type!: TParsed; + readonly _input!: TInput; + readonly _kind!: TKind; + readonly _domain!: TDomain; + + constructor( + private readonly parser: SchemaParser, + private readonly jsonSchema: () => PromptJsonSchema, + public readonly isOptional = false, + public readonly templateInfo?: PromptSchemaTemplateInfo, + ) {} + + parse(value: unknown, path = "value", root: unknown = value): TParsed { + return this.parser(value, path, root); + } + + toJSONSchema(): PromptJsonSchema { + return this.jsonSchema(); + } + + optional(): PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + > { + return new PromptSchema< + TParsed | undefined, + TInput | undefined, + TKind, + TDomain + >( + (value, path, root) => + value === undefined ? undefined : this.parser(value, path, root), + () => this.jsonSchema(), + true, + this.templateInfo, + ); + } +} + +type AnySchema = PromptSchema; +type InputSchema = PromptSchema; +type OutputSchema = PromptSchema; + +type InferSchema = + TSchema extends PromptSchema + ? TParsed + : never; + +type InferInputSchema = + TSchema extends PromptSchema + ? TInput + : never; + +type SchemaShape = Record; +type InputSchemaShape = Record; +type OutputSchemaShape = Record; + +type OptionalParsedKeys = { + [K in keyof TShape]: undefined extends InferSchema ? K : never; +}[keyof TShape]; + +type OptionalInputKeys = { + [K in keyof TShape]: undefined extends InferObjectInputSchema< + TShape[K], + TShape, + K + > + ? K + : never; +}[keyof TShape]; + +type InferParsedObject = { + [K in keyof TShape as K extends OptionalParsedKeys + ? never + : K]: InferSchema; +} & { + [K in OptionalParsedKeys]?: Exclude< + InferSchema, + undefined + >; +}; + +type InferInputObject = { + [K in keyof TShape as K extends OptionalInputKeys + ? never + : K]: InferObjectInputSchema; +} & { + [K in OptionalInputKeys]?: Exclude< + InferObjectInputSchema, + undefined + >; +}; + +type InferObjectInputSchema< + TSchema extends AnySchema, + TShape extends SchemaShape, + TKey extends keyof TShape, +> = + TSchema extends PromptSchema + ? TKind extends PromptFieldKind + ? PromptInputValueForObject< + TDefinition, + TPromptKind, + TShape, + TKey, + TInput + > + : TInput + : never; + +const builtPromptMarker = Symbol("braintrust.experimental_prompt.built"); +const promptDefinitionMarker = Symbol( + "braintrust.experimental_prompt.definition", +); +const promptTextMarker = Symbol("braintrust.experimental_prompt.text"); +const promptFileMarker = Symbol("braintrust.experimental_prompt.file"); +const promptDependencyMarker = Symbol( + "braintrust.experimental_prompt.dependencies", +); +const mustacheTemplateValueMarker = Symbol( + "braintrust.experimental_prompt.mustache_template_value", +); +const templateValueStateMarker = Symbol( + "braintrust.experimental_prompt.template_value_state", +); + +type PromptRole = "system" | "user" | "assistant"; + +export type InlineAttachmentReference = { + type: "inline_attachment"; + src: string; + content_type?: string; + filename?: string; + data?: string; +}; + +export type PromptAttachment = + | string + | Blob + | ArrayBuffer + | ArrayBufferView + | BaseAttachment + | ReadonlyAttachment + | AttachmentReference + | InlineAttachmentReference; + +type PromptFileOptions = { + filename?: string; + contentType?: string; + detail?: "auto" | "low" | "high"; +}; + +export type PromptTextContentPart = { + type: "text"; + text: string; +}; + +export type PromptFileContentPart = { + readonly [promptFileMarker]: true; + type: "file"; + file: { + value: unknown; + filename?: string; + contentType?: string; + detail?: "auto" | "low" | "high"; + }; +}; + +export type PromptMessageContentPart = + | PromptTextContentPart + | PromptFileContentPart; + +type PromptUserContentPartInput = + | string + | PromptText + | PromptTextContentPart + | PromptFileContentPart; + +export type PromptMessage = { + role: PromptRole; + content: string | PromptMessageContentPart[]; +}; + +type PromptMessageWithDependencies = PromptMessage & { + readonly [promptDependencyMarker]?: PromptDependencyEntry[]; +}; + +type PromptText = { + readonly [promptTextMarker]: true; + readonly content: string; + readonly dependencies: PromptDependencyEntry[]; +}; + +type PromptVariableMode = "runtime" | "mustache"; + +type TemplateValueState = { + readonly path: string; + readonly mode: PromptVariableMode; + readonly runtimeValue?: unknown; + readonly sectionPath?: string; + readonly relativePath?: string; +}; + +type MustacheTemplateValue = { + readonly [mustacheTemplateValueMarker]: true; + readonly [templateValueStateMarker]: TemplateValueState; +}; + +type PromptDependencyEntry = { + id?: string; + slug: string; + name?: string; + version?: string; + role: "root" | "include"; + parent?: string; + input: unknown; +}; + +export type PromptDependencies = { + root: { + id?: string; + slug: string; + name?: string; + version?: string; + }; + prompts: PromptDependencyEntry[]; +}; + +export type ExperimentalPromptData = { + id?: string; + slug: string; + name?: string; + version?: string; + model?: string; + inputSchema: PromptJsonSchema; + outputSchema?: PromptJsonSchema; + dependencies: PromptDependencies; +} & ( + | { + kind: "messages"; + messages: PromptMessage[]; + } + | { + kind: "string"; + content: string; + } +); + +type PromptTemplateResult = readonly PromptMessage[] | PromptText; + +type PromptTemplateContext = { + variables: TVariables; + include: ( + definition: TDefinition, + input: InputOf, + ) => BuiltPromptOf; +}; + +type PromptTemplateScope = { + rootKeys: ReadonlySet; + pathForKey: (key: string) => string; +}; + +type TemplateNestedPromptBuilder = ( + definition: AnyPromptDefinition, + kind: PromptKind, + fieldPath: string, +) => AnyBuiltPrompt; + +type PromptListTag = ( + strings: TemplateStringsArray, + ...values: readonly unknown[] +) => PromptText; + +type PromptTemplateField = TValue extends AnyBuiltPrompt + ? TValue + : unknown & + (TValue extends readonly (infer TItem)[] + ? { list: PromptListTag & PromptTemplateField } + : TValue extends object + ? { [K in keyof TValue]: PromptTemplateField } + : {}); + +type TemplateRenderContext = { + sectionPath?: string; + item?: unknown; +}; + +type InputSchemaHelpers = typeof inputSchemaHelpers; +type OutputSchemaHelpers = typeof outputSchemaHelpers; + +type PromptDefinitionOptions< + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined, + TTemplateResult extends PromptTemplateResult, +> = { + id?: string; + slug: string; + name?: string; + version?: string; + model?: string; + inputSchema: (s: InputSchemaHelpers) => TInputSchema; + outputSchema?: (s: OutputSchemaHelpers) => TOutputSchema; + template: ( + context: PromptTemplateContext< + PromptTemplateField> + >, + ) => TTemplateResult; +}; + +class PromptDefinition< + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined, + TTemplateResult extends PromptTemplateResult, +> { + readonly [promptDefinitionMarker] = true; + readonly id?: string; + readonly slug: string; + readonly name?: string; + readonly version?: string; + readonly model?: string; + readonly inputSchema: TInputSchema; + readonly outputSchema?: TOutputSchema; + + private readonly template: PromptDefinitionOptions< + TInputSchema, + TOutputSchema, + TTemplateResult + >["template"]; + + constructor( + opts: PromptDefinitionOptions, + ) { + this.id = opts.id; + this.slug = opts.slug; + this.name = opts.name; + this.version = opts.version; + this.model = opts.model; + this.inputSchema = opts.inputSchema(inputSchemaHelpers); + this.outputSchema = opts.outputSchema?.(outputSchemaHelpers); + this.template = opts.template; + } + + build( + input: InferInputSchema, + ): BuiltPromptForTemplateResult< + InferSchema, + InferOutput, + TTemplateResult + > { + const parsedInput = this.inputSchema.parse( + input, + "input", + input, + ) as InferSchema; + const variables = createPromptVariables( + this.inputSchema, + createRootTemplateScope(this.inputSchema), + "runtime", + parsedInput, + () => { + throw new Error("prompt variables could not resolve a built prompt"); + }, + ) as PromptTemplateField>; + const rendered = this.template({ + variables, + include: (definition, includeInput) => + definition.build(includeInput) as BuiltPromptOf, + }); + const root = { + id: this.id, + slug: this.slug, + name: this.name, + version: this.version, + }; + + if (isPromptText(rendered)) { + const dependencies = createPromptDependencies( + root, + parsedInput, + [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectDependencyEntries(rendered.dependencies, this.slug), + ], + this.inputSchema, + ); + + return new BuiltStringPrompt< + InferSchema, + InferOutput + >({ + definition: { + model: this.model, + inputSchema: this.inputSchema as PromptSchema< + InferSchema, + unknown, + unknown, + "input" + >, + outputSchema: this.outputSchema as + | PromptSchema< + InferOutput, + unknown, + unknown, + "output" + > + | undefined, + }, + input: parsedInput, + content: rendered.content, + dependencies, + }) as BuiltPromptForTemplateResult< + InferSchema, + InferOutput, + TTemplateResult + >; + } + + if (!Array.isArray(rendered)) { + throw new Error("template must return a message array or prompt.text"); + } + + const messages = rendered.map((message, index) => + assertPromptMessage(message, `template[${index}]`), + ); + const dependencies = createPromptDependencies( + root, + parsedInput, + [ + ...collectBuiltPromptDependencies(parsedInput, this.slug), + ...collectMessageDependencies(messages, this.slug), + ], + this.inputSchema, + ); + + return new BuiltMessagesPrompt< + InferSchema, + InferOutput + >({ + definition: { + model: this.model, + inputSchema: this.inputSchema as PromptSchema< + InferSchema, + unknown, + unknown, + "input" + >, + outputSchema: this.outputSchema as + | PromptSchema, unknown, unknown, "output"> + | undefined, + }, + input: parsedInput, + messages, + dependencies, + }) as BuiltPromptForTemplateResult< + InferSchema, + InferOutput, + TTemplateResult + >; + } + + toPromptData(): ExperimentalPromptData { + return this.compileTemplate(createRootTemplateScope(this.inputSchema)); + } + + private compileTemplate(scope: PromptTemplateScope): ExperimentalPromptData { + const variables = createPromptVariables( + this.inputSchema, + scope, + "mustache", + undefined, + (definition, kind, fieldPath) => { + const nested = definition.compileTemplate( + createNestedTemplateScope( + definition.inputSchema, + scope.rootKeys, + fieldPath, + ), + ); + return templateDataToBuiltPrompt(nested, kind); + }, + ) as PromptTemplateField>; + const rendered = this.template({ + variables, + include: (definition) => { + const nested = definition.compileTemplate( + createRootTemplateScope(definition.inputSchema), + ); + return templateDataToBuiltPrompt(nested, nested.kind) as BuiltPromptOf< + typeof definition + >; + }, + }); + const root = promptDefinitionRoot(this); + const inputSnapshot = createTemplateDependencyInput( + this.inputSchema, + scope, + ); + + if (isPromptText(rendered)) { + const dependencies = createPromptDependencies(root, inputSnapshot, [ + ...collectBuiltPromptDependencies(variables, this.slug), + ...collectDependencyEntries(rendered.dependencies, this.slug), + ]); + + return { + ...root, + model: this.model, + inputSchema: this.inputSchema.toJSONSchema(), + outputSchema: this.outputSchema?.toJSONSchema(), + dependencies, + kind: "string", + content: rendered.content, + }; + } + + if (!Array.isArray(rendered)) { + throw new Error("template must return a message array or prompt.text"); + } + + const messages = rendered.map((message, index) => + assertPromptMessage(message, `template[${index}]`), + ); + const dependencies = createPromptDependencies(root, inputSnapshot, [ + ...collectBuiltPromptDependencies(variables, this.slug), + ...collectMessageDependencies(messages, this.slug), + ]); + + return { + ...root, + model: this.model, + inputSchema: this.inputSchema.toJSONSchema(), + outputSchema: this.outputSchema?.toJSONSchema(), + dependencies, + kind: "messages", + messages, + }; + } +} + +type AnyPromptDefinition = PromptDefinition< + InputSchema, + OutputSchema | undefined, + PromptTemplateResult +>; + +type AnyMessagesPromptDefinition = PromptDefinition< + InputSchema, + OutputSchema | undefined, + readonly PromptMessage[] +>; + +type AnyStringPromptDefinition = PromptDefinition< + InputSchema, + OutputSchema | undefined, + PromptText +>; + +type InferOutput = + TOutputSchema extends PromptSchema + ? TParsed + : unknown; + +type InputOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + OutputSchema | undefined, + PromptTemplateResult + > + ? InferInputSchema + : never; + +type ParsedInputOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + OutputSchema | undefined, + PromptTemplateResult + > + ? InferSchema + : never; + +type OutputOf = + TDefinition extends PromptDefinition< + InputSchema, + infer TOutputSchema, + PromptTemplateResult + > + ? InferOutput + : never; + +type BuiltPromptOf = + TDefinition extends PromptDefinition< + infer TInputSchema, + infer TOutputSchema, + infer TTemplateResult + > + ? BuiltPromptForTemplateResult< + InferSchema, + InferOutput, + TTemplateResult + > + : never; + +type BuiltPromptForTemplateResult = + TTemplateResult extends PromptText + ? BuiltStringPrompt + : TTemplateResult extends readonly PromptMessage[] + ? BuiltMessagesPrompt + : never; + +type BuiltPromptForKind< + TInput, + TOutput, + TKind extends PromptKind, +> = TKind extends "messages" + ? BuiltMessagesPrompt + : BuiltStringPrompt; + +type PromptFieldKind< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, +> = { + type: "prompt"; + definition: TDefinition; + promptKind: TPromptKind; +}; + +type PromptInputOverrides< + TInput, + TParentKeys extends PropertyKey, +> = TInput extends object + ? Omit & + Partial>> + : TInput; + +type PromptInputValue< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, +> = + | BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TPromptKind + > + | InputOf; + +type PromptInputValueForObject< + TDefinition extends AnyPromptDefinition, + TPromptKind extends PromptKind, + TShape extends SchemaShape, + TKey extends keyof TShape, + TInput, +> = + | BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TPromptKind + > + | PromptInputOverrides, Exclude> + | (undefined extends TInput ? undefined : never); + +type BuiltPromptOptions = { + definition: { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + }; + input: TInput; + dependencies: PromptDependencies; +}; + +type BuiltMessagesPromptOptions = BuiltPromptOptions< + TInput, + TOutput +> & { + messages: PromptMessage[]; +}; + +type BuiltStringPromptOptions = BuiltPromptOptions< + TInput, + TOutput +> & { + content: string; +}; + +type PromptAdapterInput = { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + input: TInput; + dependencies: PromptDependencies; +} & ( + | { + kind: "messages"; + messages: PromptMessage[]; + } + | { + kind: "string"; + content: string; + messages: PromptMessage[]; + } +); + +type Simplify = { [K in keyof T]: T[K] } & {}; + +type DeepMerge = Simplify< + Omit & { + [K in keyof TExtension]: K extends keyof TBase + ? DeepMergeValue + : TExtension[K]; + } +>; + +type DeepMergeValue = TBase extends readonly unknown[] + ? TExtension + : TExtension extends readonly unknown[] + ? TExtension + : TBase extends object + ? TExtension extends object + ? DeepMerge + : TExtension + : TExtension; + +type PromptExtension = Record; +type PromptAdapterResult = PromptExtension; +type MaybePromise = T | Promise; + +type Extendable = T & { + extend( + extension: TExtension, + ): Extendable>; +}; + +type PromptAdapter = ( + builtPrompt: PromptAdapterInput, +) => MaybePromise; + +type PromptAdapterToResult = + | Extendable + | Promise>; + +class BuiltMessagesPrompt implements Iterable { + readonly [builtPromptMarker] = true; + readonly kind = "messages"; + readonly definition: { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + }; + readonly input: TInput; + readonly messages: PromptMessage[]; + readonly dependencies: PromptDependencies; + + constructor(opts: BuiltMessagesPromptOptions) { + this.definition = opts.definition; + this.input = opts.input; + this.messages = opts.messages; + this.dependencies = opts.dependencies; + } + + [Symbol.iterator](): Iterator { + return this.messages + .map((message) => + attachDependenciesToMessage(message, this.dependencies.prompts), + ) + [Symbol.iterator](); + } + + to( + adapter: PromptAdapter, + ): PromptAdapterToResult { + const result = adapter({ + kind: "messages", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + messages: this.messages, + dependencies: this.dependencies, + }); + return isPromiseLike(result) + ? result.then((resolved) => makeExtendableAdapterResult(resolved)) + : makeExtendableAdapterResult(result); + } +} + +class BuiltStringPrompt { + readonly [builtPromptMarker] = true; + readonly kind = "string"; + readonly definition: { + model?: string; + inputSchema: PromptSchema; + outputSchema?: PromptSchema; + }; + readonly input: TInput; + readonly content: string; + readonly dependencies: PromptDependencies; + + constructor(opts: BuiltStringPromptOptions) { + this.definition = opts.definition; + this.input = opts.input; + this.content = opts.content; + this.dependencies = opts.dependencies; + } + + to( + adapter: PromptAdapter, + ): PromptAdapterToResult { + const result = adapter({ + kind: "string", + model: this.definition.model, + inputSchema: this.definition.inputSchema, + outputSchema: this.definition.outputSchema, + input: this.input, + content: this.content, + messages: [{ role: "user", content: this.content }], + dependencies: this.dependencies, + }); + return isPromiseLike(result) + ? result.then((resolved) => makeExtendableAdapterResult(resolved)) + : makeExtendableAdapterResult(result); + } +} + +function isPromiseLike(value: MaybePromise): value is Promise { + return isRecord(value) && typeof value.then === "function"; +} + +function makeExtendableAdapterResult( + result: TResult, +): Extendable { + if (!isMergeableObject(result)) { + throw new Error("prompt adapters must return an object"); + } + + const extendable = result as Extendable; + Object.defineProperty(extendable, "extend", { + value: (extension: TExtension) => { + if (!isMergeableObject(extension)) { + throw new Error("extend must receive an object"); + } + return makeExtendableAdapterResult( + deepMergeObjects(extendable, extension) as DeepMerge< + TResult, + TExtension + >, + ); + }, + enumerable: false, + configurable: true, + }); + return extendable; +} + +function deepMergeObjects( + base: PromptExtension, + extension: PromptExtension, +): PromptExtension { + const merged: PromptExtension = { ...base }; + for (const [key, value] of Object.entries(extension)) { + const baseValue = merged[key]; + merged[key] = + isMergeableObject(baseValue) && isMergeableObject(value) + ? deepMergeObjects(baseValue, value) + : value; + } + return merged; +} + +function isMergeableObject(value: unknown): value is PromptExtension { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +type AnyBuiltPrompt = + | BuiltMessagesPrompt + | BuiltStringPrompt; + +function definePrompt< + TInputSchema extends InputSchema, + TOutputSchema extends OutputSchema | undefined = undefined, + TTemplateResult extends PromptTemplateResult = PromptTemplateResult, +>( + opts: PromptDefinitionOptions, +): PromptDefinition { + return new PromptDefinition(opts); +} + +type PromptMessageTag = ( + strings: TemplateStringsArray, + ...values: readonly unknown[] +) => PromptMessage; + +type PromptUserMessageTag = PromptMessageTag & { + (content: readonly PromptUserContentPartInput[]): PromptMessage; +}; + +function messageTag(role: "system" | "assistant"): PromptMessageTag; +function messageTag(role: "user"): PromptUserMessageTag; +function messageTag(role: PromptRole): PromptMessageTag | PromptUserMessageTag { + return (( + stringsOrContent: + | TemplateStringsArray + | readonly PromptUserContentPartInput[], + ...values: readonly unknown[] + ): PromptMessage => { + if (!isTemplateStringsArray(stringsOrContent)) { + if (role !== "user") { + throw new Error( + "rich prompt content is only supported for user messages", + ); + } + return userMessageFromContentParts(stringsOrContent); + } + + const rendered = renderTaggedTemplate(stringsOrContent, values); + return attachDependenciesToMessage( + { role, content: rendered.content }, + rendered.dependencies, + ); + }) as PromptMessageTag | PromptUserMessageTag; +} + +function userMessageFromContentParts( + parts: readonly PromptUserContentPartInput[], +): PromptMessage { + const dependencies: PromptDependencyEntry[] = []; + const content = parts.map((part, index): PromptMessageContentPart => { + if (typeof part === "string") { + return { type: "text", text: part }; + } + if (isPromptText(part)) { + dependencies.push(...part.dependencies); + return { type: "text", text: part.content }; + } + if (isPromptFileContentPart(part)) { + return part; + } + if ( + isRecord(part) && + part.type === "text" && + typeof part.text === "string" + ) { + return { type: "text", text: part.text }; + } + + throw new Error( + `user content part ${index} must be prompt.text or prompt.file`, + ); + }); + return attachDependenciesToMessage({ role: "user", content }, dependencies); +} + +function filePart( + value: unknown, + options: PromptFileOptions = {}, +): PromptFileContentPart { + return { + [promptFileMarker]: true, + type: "file", + file: { + value, + filename: options.filename, + contentType: options.contentType, + detail: options.detail, + }, + }; +} + +function isTemplateStringsArray(value: unknown): value is TemplateStringsArray { + return ( + Array.isArray(value) && Array.isArray((value as { raw?: unknown }).raw) + ); +} + +function textTag( + strings: TemplateStringsArray, + ...values: readonly unknown[] +): PromptText { + const rendered = renderTaggedTemplate(strings, values); + return { + [promptTextMarker]: true, + content: rendered.content, + dependencies: rendered.dependencies, + }; +} + +function renderTaggedTemplate( + strings: TemplateStringsArray, + values: readonly unknown[], + context?: TemplateRenderContext, +): { content: string; dependencies: PromptDependencyEntry[] } { + let content = strings[0] ?? ""; + const dependencies: PromptDependencyEntry[] = []; + for (let i = 0; i < values.length; i++) { + const rendered = stringifyTemplateValue(values[i], context); + content += rendered.content + (strings[i + 1] ?? ""); + dependencies.push(...rendered.dependencies); + } + return { content, dependencies }; +} + +function stringifyTemplateValue( + value: unknown, + context?: TemplateRenderContext, +): { + content: string; + dependencies: PromptDependencyEntry[]; +}; +function stringifyTemplateValue( + value: unknown, + context?: TemplateRenderContext, +): { + content: string; + dependencies: PromptDependencyEntry[]; +} { + if (value === undefined || value === null) { + return { content: "", dependencies: [] }; + } + if (isMustacheTemplateValue(value)) { + const state = value[templateValueStateMarker]; + if (state.mode === "mustache") { + if (context?.sectionPath && state.sectionPath === context.sectionPath) { + return { + content: `{{${state.relativePath ?? "."}}}`, + dependencies: [], + }; + } + return { content: `{{${state.path}}}`, dependencies: [] }; + } + + const runtimeValue = + context?.sectionPath && state.sectionPath === context.sectionPath + ? getPathValue(context.item, state.relativePath) + : state.runtimeValue; + return { content: stringifyRuntimeValue(runtimeValue), dependencies: [] }; + } + if (isBuiltStringPrompt(value)) { + return { content: value.content, dependencies: value.dependencies.prompts }; + } + if (isBuiltMessagesPrompt(value)) { + throw new Error("message prompts cannot be interpolated as text"); + } + if (isPromptText(value)) { + return { content: value.content, dependencies: value.dependencies }; + } + if (isPromptDefinition(value)) { + return { + content: `[prompt:${value.slug}${value.version ? `@${value.version}` : ""}]`, + dependencies: [], + }; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return { content: String(value), dependencies: [] }; + } + return { content: JSON.stringify(value), dependencies: [] }; +} + +function attachDependenciesToMessage( + message: PromptMessage, + dependencies: PromptDependencyEntry[], +): PromptMessage { + if (dependencies.length === 0) { + return message; + } + const messageWithDependencies: PromptMessageWithDependencies = { ...message }; + Object.defineProperty(messageWithDependencies, promptDependencyMarker, { + value: dependencies, + enumerable: false, + }); + return messageWithDependencies; +} + +function assertPromptMessage(value: unknown, path: string): PromptMessage { + if ( + !isRecord(value) || + (value.role !== "system" && + value.role !== "user" && + value.role !== "assistant") || + !isPromptMessageContent(value.role, value.content) + ) { + throw new Error(`${path} must be a prompt message`); + } + return value as PromptMessage; +} + +function isPromptMessageContent(role: unknown, content: unknown): boolean { + if (typeof content === "string") { + return true; + } + if (!Array.isArray(content)) { + return false; + } + if (role !== "user") { + return false; + } + return content.every(isPromptMessageContentPart); +} + +function isPromptMessageContentPart( + value: unknown, +): value is PromptMessageContentPart { + if (!isRecord(value) || typeof value.type !== "string") { + return false; + } + if (value.type === "text") { + return typeof value.text === "string"; + } + return isPromptFileContentPart(value); +} + +function createRootTemplateScope( + inputSchema: InputSchema, +): PromptTemplateScope { + return { + rootKeys: getObjectSchemaKeys(inputSchema), + pathForKey: (key) => key, + }; +} + +function createNestedTemplateScope( + inputSchema: InputSchema, + rootKeys: ReadonlySet, + fieldPath: string, +): PromptTemplateScope { + const nestedKeys = getObjectSchemaKeys(inputSchema); + const fieldKey = lastPathSegment(fieldPath); + return { + rootKeys: new Set([...rootKeys, ...nestedKeys]), + pathForKey: (key) => + rootKeys.has(key) && key !== fieldKey ? key : `${fieldPath}.${key}`, + }; +} + +function getObjectSchemaKeys(schema: InputSchema): Set { + return schema.templateInfo?.type === "object" + ? new Set(Object.keys(schema.templateInfo.shape)) + : new Set(); +} + +function lastPathSegment(path: string): string { + return path.split(".").at(-1) ?? path; +} + +function createPromptVariables( + schema: InputSchema, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + path = "input", + sectionPath?: string, + relativePath?: string, +): unknown { + if (mode === "runtime" && isBuiltPrompt(runtimeValue)) { + return runtimeValue; + } + + const templateInfo = schema.templateInfo; + if (templateInfo?.type === "object") { + return createPromptVariableObject( + templateInfo.shape, + scope, + mode, + runtimeValue, + nestedPromptBuilder, + path === "input" ? undefined : path, + sectionPath, + relativePath, + ); + } + + if (templateInfo?.type === "array") { + return createPromptVariableArray( + templateInfo.item as InputSchema, + scope, + mode, + runtimeValue, + nestedPromptBuilder, + path, + sectionPath, + relativePath, + ); + } + + if (templateInfo?.type === "promptDefinition") { + return mode === "runtime" + ? runtimeValue + : nestedPromptBuilder(templateInfo.definition, templateInfo.kind, path); + } + + if (templateInfo?.type === "attachment") { + return mode === "runtime" + ? runtimeValue + : createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }); + } + + return createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }); +} + +function createPromptVariableObject( + shape: SchemaShape, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + basePath?: string, + sectionPath?: string, + relativePath?: string, +): Record { + const variableObject: Record = {}; + if (basePath) { + attachPromptVariableValue(variableObject, { + path: basePath, + mode, + runtimeValue, + sectionPath, + relativePath, + }); + } + for (const [key, schema] of Object.entries(shape)) { + const path = basePath ? `${basePath}.${key}` : scope.pathForKey(key); + const childRelativePath = sectionPath + ? relativePath + ? `${relativePath}.${key}` + : key + : undefined; + variableObject[key] = createPromptVariables( + schema as InputSchema, + scope, + mode, + getObjectProperty(runtimeValue, key), + nestedPromptBuilder, + path, + sectionPath, + childRelativePath, + ); + } + return variableObject; +} + +function createPromptVariableArray( + itemSchema: InputSchema, + scope: PromptTemplateScope, + mode: PromptVariableMode, + runtimeValue: unknown, + nestedPromptBuilder: TemplateNestedPromptBuilder, + path: string, + sectionPath?: string, + relativePath?: string, +): Record { + const variableArray = createPromptVariableValue({ + path, + mode, + runtimeValue, + sectionPath, + relativePath, + }) as Record; + const listSectionPath = path; + const listTag = (( + strings: TemplateStringsArray, + ...values: readonly unknown[] + ): PromptText => { + if (mode === "mustache") { + const rendered = renderTaggedTemplate(strings, values, { + sectionPath: listSectionPath, + }); + const sectionName = relativePath ?? path; + return { + [promptTextMarker]: true, + content: `{{#${sectionName}}}${rendered.content}{{/${sectionName}}}`, + dependencies: rendered.dependencies, + }; + } + + const items = Array.isArray(runtimeValue) ? runtimeValue : []; + const renderedItems = items.map((item) => + renderTaggedTemplate(strings, values, { + sectionPath: listSectionPath, + item, + }), + ); + return { + [promptTextMarker]: true, + content: renderedItems.map((item) => item.content).join(""), + dependencies: renderedItems.flatMap((item) => item.dependencies), + }; + }) as PromptListTag & Record; + + attachPromptVariableValue(listTag, { + path, + mode, + runtimeValue, + sectionPath: listSectionPath, + relativePath: undefined, + }); + const itemVariables = createPromptVariables( + itemSchema, + scope, + mode, + undefined, + nestedPromptBuilder, + path, + listSectionPath, + ); + if (isRecord(itemVariables)) { + for (const key of Object.keys(itemVariables)) { + Object.defineProperty( + listTag, + key, + Object.getOwnPropertyDescriptor(itemVariables, key) ?? { + value: itemVariables[key], + enumerable: true, + }, + ); + } + } + + Object.defineProperty(variableArray, "list", { + value: listTag, + enumerable: true, + }); + return variableArray; +} + +function createTemplateDependencyInput( + schema: InputSchema, + scope: PromptTemplateScope, + path = "input", +): unknown { + const templateInfo = schema.templateInfo; + if (templateInfo?.type === "object") { + return Object.fromEntries( + Object.entries(templateInfo.shape).map(([key, childSchema]) => { + const childPath = + path === "input" ? scope.pathForKey(key) : `${path}.${key}`; + return [ + key, + createTemplateDependencyInput( + childSchema as InputSchema, + scope, + childPath, + ), + ]; + }), + ); + } + + if (templateInfo?.type === "array") { + return `{{${path}}}`; + } + + if (templateInfo?.type === "promptDefinition") { + return { + type: + templateInfo.kind === "messages" + ? "template_messages_prompt" + : "template_string_prompt", + root: promptDefinitionRoot(templateInfo.definition), + }; + } + + return `{{${path}}}`; +} + +function createPromptVariableValue( + state: TemplateValueState, +): MustacheTemplateValue { + return attachPromptVariableValue({}, state) as MustacheTemplateValue; +} + +function attachPromptVariableValue( + value: T, + state: TemplateValueState, +): T { + Object.defineProperties(value, { + [mustacheTemplateValueMarker]: { + value: true, + enumerable: false, + }, + [templateValueStateMarker]: { + value: state, + enumerable: false, + }, + toString: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + valueOf: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + [Symbol.toPrimitive]: { + value: () => stringifyTemplateValue(value).content, + enumerable: false, + }, + }); + return value; +} + +function getObjectProperty(value: unknown, key: string): unknown { + if (isRecord(value)) { + return value[key]; + } + if (Array.isArray(value)) { + return value[Number(key)]; + } + return undefined; +} + +function getPathValue(value: unknown, path?: string): unknown { + if (!path) { + return value; + } + return path + .split(".") + .reduce((current, key) => getObjectProperty(current, key), value); +} + +function stringifyRuntimeValue(value: unknown): string { + if (value === undefined || value === null) { + return ""; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + return String(value); + } + return JSON.stringify(value); +} + +function isMustacheTemplateValue( + value: unknown, +): value is MustacheTemplateValue { + return ( + typeof value === "object" && + value !== null && + mustacheTemplateValueMarker in value + ); +} + +function promptDefinitionRoot( + definition: AnyPromptDefinition, +): PromptDependencies["root"] { + return { + id: definition.id, + slug: definition.slug, + name: definition.name, + version: definition.version, + }; +} + +function templateDataToBuiltPrompt( + data: ExperimentalPromptData, + kind: PromptKind, +): AnyBuiltPrompt { + if (data.kind !== kind) { + const label = kind === "messages" ? "messages" : "string"; + throw new Error(`template prompt must be a ${label} prompt`); + } + + const definition = { + model: data.model, + inputSchema: unknownSchema(), + outputSchema: undefined, + }; + if (data.kind === "messages") { + return new BuiltMessagesPrompt({ + definition, + input: data.dependencies.prompts[0]?.input, + messages: data.messages, + dependencies: data.dependencies, + }); + } + return new BuiltStringPrompt({ + definition, + input: data.dependencies.prompts[0]?.input, + content: data.content, + dependencies: data.dependencies, + }); +} + +function createPromptDependencies( + root: PromptDependencies["root"], + input: unknown, + entries: PromptDependencyEntry[], + inputSchema?: InputSchema, +): PromptDependencies { + return { + root, + prompts: [ + { + ...root, + role: "root", + input: sanitizeDependencyInput(input, inputSchema), + }, + ...dedupeDependencyEntries(entries), + ], + }; +} + +function collectDependencyEntries( + entries: readonly PromptDependencyEntry[], + parent: string, +): PromptDependencyEntry[] { + return entries.map((entry) => ({ + ...entry, + role: "include" as const, + parent, + })); +} + +function collectMessageDependencies( + messages: readonly PromptMessage[], + parent: string, +): PromptDependencyEntry[] { + return messages.flatMap((message) => + collectDependencyEntries( + (message as PromptMessageWithDependencies)[promptDependencyMarker] ?? [], + parent, + ), + ); +} + +function collectBuiltPromptDependencies( + value: unknown, + parent: string, +): PromptDependencyEntry[] { + if (isPromptAttachmentValue(value) || isPromptFileContentPart(value)) { + return []; + } + if (isBuiltPrompt(value)) { + return collectDependencyEntries(value.dependencies.prompts, parent); + } + if (Array.isArray(value)) { + return value.flatMap((item) => + collectBuiltPromptDependencies(item, parent), + ); + } + if (isRecord(value)) { + return Object.values(value).flatMap((item) => + collectBuiltPromptDependencies(item, parent), + ); + } + return []; +} + +function dedupeDependencyEntries( + entries: PromptDependencyEntry[], +): PromptDependencyEntry[] { + const seen = new Set(); + return entries.filter((entry) => { + const key = JSON.stringify(entry); + if (seen.has(key)) { + return false; + } + seen.add(key); + return true; + }); +} + +function sanitizeDependencyInput( + value: unknown, + schema?: InputSchema, +): unknown { + const templateInfo = schema?.templateInfo; + if (templateInfo?.type === "attachment") { + return summarizeAttachmentInput(value); + } + if (templateInfo?.type === "object" && isRecord(value)) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + sanitizeDependencyInput(item, templateInfo.shape[key] as InputSchema), + ]), + ); + } + if (templateInfo?.type === "array" && Array.isArray(value)) { + return value.map((item) => + sanitizeDependencyInput(item, templateInfo.item as InputSchema), + ); + } + if (isBuiltPrompt(value)) { + return { + type: + value.kind === "messages" + ? "built_messages_prompt" + : "built_string_prompt", + root: value.dependencies.root, + }; + } + if (isPromptDefinition(value)) { + return { + type: "prompt_definition", + slug: value.slug, + version: value.version, + }; + } + if (isPromptFileContentPart(value)) { + return { + type: "prompt_file", + file: summarizeAttachmentInput(value.file.value), + filename: value.file.filename, + content_type: value.file.contentType, + }; + } + if (isPromptAttachmentValue(value)) { + return summarizeAttachmentInput(value); + } + if (typeof value === "string" && value.startsWith("data:")) { + return summarizeStringAttachment(value); + } + if (Array.isArray(value)) { + return value.map((item) => sanitizeDependencyInput(item)); + } + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + key, + sanitizeDependencyInput(item), + ]), + ); + } + return value; +} + +function summarizeAttachmentInput(value: unknown): unknown { + if (value instanceof BaseAttachment || value instanceof ReadonlyAttachment) { + return { + type: "attachment", + reference: value.reference, + }; + } + if (isAttachmentReference(value)) { + return value; + } + if (isInlineAttachmentReference(value)) { + return { + type: "inline_attachment", + content_type: value.content_type, + filename: value.filename, + src: summarizeStringAttachment(value.src), + }; + } + if (isBlob(value)) { + return { + type: "blob", + content_type: value.type || undefined, + byte_length: value.size, + }; + } + if (value instanceof ArrayBuffer) { + return { type: "array_buffer", byte_length: value.byteLength }; + } + if (ArrayBuffer.isView(value)) { + return { + type: "binary", + byte_length: value.byteLength, + }; + } + if (typeof value === "string") { + return summarizeStringAttachment(value); + } + return { type: "attachment", value_type: typeof value }; +} + +function summarizeStringAttachment(value: string): unknown { + if (value.startsWith("data:")) { + return { + type: "data_url", + content_type: dataUrlContentType(value), + byte_length: value.length, + }; + } + return value; +} + +function isBuiltPrompt(value: unknown): value is AnyBuiltPrompt { + return ( + typeof value === "object" && value !== null && builtPromptMarker in value + ); +} + +function isBuiltMessagesPrompt( + value: unknown, +): value is BuiltMessagesPrompt { + return isBuiltPrompt(value) && value.kind === "messages"; +} + +function isBuiltStringPrompt( + value: unknown, +): value is BuiltStringPrompt { + return isBuiltPrompt(value) && value.kind === "string"; +} + +function isPromptDefinition(value: unknown): value is AnyPromptDefinition { + return ( + typeof value === "object" && + value !== null && + promptDefinitionMarker in value + ); +} + +function isPromptText(value: unknown): value is PromptText { + return ( + typeof value === "object" && value !== null && promptTextMarker in value + ); +} + +function isPromptFileContentPart( + value: unknown, +): value is PromptFileContentPart { + return ( + typeof value === "object" && value !== null && promptFileMarker in value + ); +} + +function isPromptAttachmentValue(value: unknown): value is PromptAttachment { + return ( + typeof value === "string" || + isBlob(value) || + value instanceof ArrayBuffer || + ArrayBuffer.isView(value) || + value instanceof BaseAttachment || + value instanceof ReadonlyAttachment || + isAttachmentReference(value) || + isInlineAttachmentReference(value) + ); +} + +function isBlob(value: unknown): value is Blob { + return typeof Blob !== "undefined" && value instanceof Blob; +} + +function isAttachmentReference(value: unknown): value is AttachmentReference { + return ( + isRecord(value) && + ((value.type === "braintrust_attachment" && + typeof value.key === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string") || + (value.type === "external_attachment" && + typeof value.url === "string" && + typeof value.filename === "string" && + typeof value.content_type === "string")) + ); +} + +function isInlineAttachmentReference( + value: unknown, +): value is InlineAttachmentReference { + return ( + isRecord(value) && + value.type === "inline_attachment" && + typeof value.src === "string" && + (value.content_type === undefined || + typeof value.content_type === "string") && + (value.filename === undefined || typeof value.filename === "string") && + (value.data === undefined || typeof value.data === "string") + ); +} + +function dataUrlContentType(value: string): string | undefined { + return value.match(/^data:([^;,]+)[;,]/)?.[1]; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function mergePromptInputs(parent: unknown, overrides: unknown): unknown { + const parentInput = isRecord(parent) ? parent : {}; + if (overrides === undefined) { + return parentInput; + } + if (isRecord(overrides)) { + return { ...parentInput, ...overrides }; + } + return overrides; +} + +function buildAnyPrompt( + definition: AnyPromptDefinition, + input: unknown, +): AnyBuiltPrompt { + return definition.build(input as never) as AnyBuiltPrompt; +} + +function stringSchema(): PromptSchema< + string, + string, + unknown, + TDomain +> { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string") { + throw new Error(`${path} must be a string`); + } + return value; + }, + () => ({ type: "string" }), + ); +} + +function numberSchema(): PromptSchema< + number, + number, + unknown, + TDomain +> { + return new PromptSchema( + (value, path) => { + if (typeof value !== "number") { + throw new Error(`${path} must be a number`); + } + return value; + }, + () => ({ type: "number" }), + ); +} + +function booleanSchema(): PromptSchema< + boolean, + boolean, + unknown, + TDomain +> { + return new PromptSchema( + (value, path) => { + if (typeof value !== "boolean") { + throw new Error(`${path} must be a boolean`); + } + return value; + }, + () => ({ type: "boolean" }), + ); +} + +function enumSchema< + const TValues extends readonly [string, ...string[]], + TDomain extends SchemaDomain = "input", +>( + values: TValues, +): PromptSchema { + return new PromptSchema( + (value, path) => { + if (typeof value !== "string" || !values.includes(value)) { + throw new Error(`${path} must be one of ${values.join(", ")}`); + } + return value; + }, + () => ({ type: "string", enum: [...values] }), + ); +} + +function createArraySchema< + TItemSchema extends AnySchema, + TDomain extends SchemaDomain, +>( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain +> { + return new PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + TDomain + >( + (value, path, root) => { + if (!Array.isArray(value)) { + throw new Error(`${path} must be an array`); + } + return value.map((itemValue, index) => + item.parse(itemValue, `${path}[${index}]`, root), + ) as InferSchema[]; + }, + () => ({ type: "array", items: item.toJSONSchema() }), + false, + { type: "array", item }, + ); +} + +function arraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "input" +> { + return createArraySchema(item); +} + +function outputArraySchema( + item: TItemSchema, +): PromptSchema< + InferSchema[], + InferInputSchema[], + unknown, + "output" +> { + return createArraySchema(item); +} + +function createObjectSchema< + TShape extends SchemaShape, + TDomain extends SchemaDomain, +>( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain +> { + return new PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + TDomain + >( + (value, path, root) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + const record = value as Record; + const rootInput = root === undefined ? record : root; + return Object.fromEntries( + Object.entries(shape) + .filter(([key, schema]) => key in record || !schema.isOptional) + .map(([key, schema]) => [ + key, + schema.parse(record[key], `${path}.${key}`, rootInput), + ]), + ) as InferParsedObject; + }, + () => ({ + type: "object", + properties: Object.fromEntries( + Object.entries(shape).map(([key, schema]) => [ + key, + schema.toJSONSchema(), + ]), + ), + required: Object.entries(shape) + .filter(([, schema]) => !schema.isOptional) + .map(([key]) => key), + additionalProperties: false, + }), + false, + { type: "object", shape }, + ); +} + +function objectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "input" +> { + return createObjectSchema(shape); +} + +function outputObjectSchema( + shape: TShape, +): PromptSchema< + InferParsedObject, + InferInputObject, + unknown, + "output" +> { + return createObjectSchema(shape); +} + +function unknownSchema(): PromptSchema< + unknown, + unknown, + unknown, + TDomain +> { + return new PromptSchema( + (value) => value, + () => ({}), + ); +} + +function attachmentSchema(): PromptSchema< + PromptAttachment, + PromptAttachment, + unknown, + "input" +> { + return new PromptSchema( + (value, path) => { + if (!isPromptAttachmentValue(value)) { + throw new Error(`${path} must be an attachment`); + } + return value; + }, + () => ({ "x-bt-type": "attachment" }), + false, + { type: "attachment" }, + ); +} + +function builtMessagesPromptSchema(): PromptSchema< + BuiltMessagesPrompt, + BuiltMessagesPrompt, + unknown, + "input" +> { + return builtPromptSchema("messages"); +} + +function builtStringPromptSchema(): PromptSchema< + BuiltStringPrompt, + BuiltStringPrompt, + unknown, + "input" +> { + return builtPromptSchema("string"); +} + +function messagesPromptDefinitionSchema< + TDefinition extends AnyMessagesPromptDefinition, +>( + definition: TDefinition, +): PromptSchema< + BuiltMessagesPrompt, OutputOf>, + PromptInputValue, + PromptFieldKind, + "input" +> { + return promptDefinitionSchema("messages", definition); +} + +function stringPromptDefinitionSchema< + TDefinition extends AnyStringPromptDefinition, +>( + definition: TDefinition, +): PromptSchema< + BuiltStringPrompt, OutputOf>, + PromptInputValue, + PromptFieldKind, + "input" +> { + return promptDefinitionSchema("string", definition); +} + +function builtPromptSchema( + kind: TKind, +): PromptSchema< + BuiltPromptForKind, + BuiltPromptForKind, + unknown, + "input" +> { + const label = kind === "messages" ? "messages" : "string"; + return new PromptSchema( + (value, path) => { + if (isBuiltPrompt(value)) { + if (value.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return value as BuiltPromptForKind; + } + + throw new Error(`${path} must be a built ${label} prompt`); + }, + () => ({ + type: "object", + "x-bt-type": + kind === "messages" ? "built_messages_prompt" : "built_string_prompt", + }), + ); +} + +function promptDefinitionSchema< + TDefinition extends AnyPromptDefinition, + TKind extends PromptKind, +>( + kind: TKind, + definition: TDefinition, +): PromptSchema< + BuiltPromptForKind, OutputOf, TKind>, + PromptInputValue, + PromptFieldKind, + "input" +> { + const label = kind === "messages" ? "messages" : "string"; + return new PromptSchema( + (value, path, root) => { + if (isBuiltPrompt(value)) { + if (value.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return value as BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TKind + >; + } + + if ( + isPromptDefinition(value) || + (isRecord(value) && isPromptDefinition(value.prompt)) + ) { + throw new Error( + `${path} must be a built ${label} prompt or prompt input`, + ); + } + + const built = buildAnyPrompt(definition, mergePromptInputs(root, value)); + if (built.kind !== kind) { + throw new Error(`${path} must be a built ${label} prompt`); + } + return built as BuiltPromptForKind< + ParsedInputOf, + OutputOf, + TKind + >; + }, + () => ({ + type: "object", + "x-bt-type": + kind === "messages" ? "built_messages_prompt" : "built_string_prompt", + }), + false, + { type: "promptDefinition", definition, kind }, + ); +} + +/** + * @internal Converts experimental prompt template data into the existing prompt + * definition shape. This is intended for future backend-saving code paths. + */ +export function promptDefinitionToMustache( + data: ExperimentalPromptData, +): MustachePromptDefinition { + if (!data.model) { + throw new Error("Cannot convert prompt data to mustache without a model"); + } + + if (data.kind === "messages") { + return { + model: data.model, + messages: data.messages.map(promptMessageToMustacheMessage), + }; + } + + return { + model: data.model, + messages: [{ role: "user", content: data.content }], + }; +} + +function promptMessageToMustacheMessage( + message: PromptMessage, +): ChatCompletionMessageParamType { + if (typeof message.content === "string") { + if (message.role === "system") { + return { role: "system", content: message.content }; + } + if (message.role === "assistant") { + return { role: "assistant", content: message.content }; + } + return { role: "user", content: message.content }; + } + if (message.role !== "user") { + throw new Error("Only user messages can contain prompt.file parts"); + } + return { + role: "user", + content: message.content.map(promptContentPartToMustachePart), + }; +} + +function promptContentPartToMustachePart( + part: PromptMessageContentPart, +): ChatCompletionContentPartType { + if (part.type === "text") { + return part; + } + + const value = stringifyTemplateValue(part.file.value).content; + const contentType = + part.file.contentType ?? + (typeof value === "string" ? dataUrlContentType(value) : undefined); + if (isImageContentType(contentType)) { + return { + type: "image_url" as const, + image_url: { + url: value, + ...(part.file.detail ? { detail: part.file.detail } : undefined), + }, + }; + } + + return { + type: "file" as const, + file: { + file_data: value, + ...(part.file.filename ? { filename: part.file.filename } : undefined), + }, + }; +} + +function isImageContentType(contentType: string | undefined): boolean { + return contentType?.startsWith("image/") ?? false; +} + +const inputSchemaHelpers = { + string: stringSchema, + number: numberSchema, + boolean: booleanSchema, + enum: enumSchema, + array: arraySchema, + object: objectSchema, + unknown: unknownSchema, + attachment: attachmentSchema, + builtMessagesPrompt: builtMessagesPromptSchema, + builtStringPrompt: builtStringPromptSchema, + messagesPromptDefinition: messagesPromptDefinitionSchema, + stringPromptDefinition: stringPromptDefinitionSchema, +}; + +const outputSchemaHelpers = { + string: () => stringSchema<"output">(), + number: () => numberSchema<"output">(), + boolean: () => booleanSchema<"output">(), + enum: ( + values: TValues, + ) => enumSchema(values), + array: outputArraySchema, + object: outputObjectSchema, + unknown: () => unknownSchema<"output">(), +}; + +export const prompt = { + define: definePrompt, + system: messageTag("system"), + user: messageTag("user"), + assistant: messageTag("assistant"), + text: textTag, + file: filePart, + isBuiltPrompt, + isPromptDefinition, + adapters, +}; diff --git a/js/src/exports.ts b/js/src/exports.ts index 54cd03e2a..12e469ca1 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -262,6 +262,8 @@ export { PromptDefinitionWithTools, } from "./prompt-schemas"; +export { prompt } from "./experimental-prompt-api"; + export type { Trace, SpanData, GetThreadOptions } from "./trace"; export { SpanFetcher, CachedSpanFetcher, LocalTrace } from "./trace"; diff --git a/knip.jsonc b/knip.jsonc index 99bf84691..3775d1955 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -44,6 +44,7 @@ "ignoreDependencies": ["openai", "openai-v4", "openai-v5"], }, "js": { + "ignore": ["promptDefinitionToMustache"], "entry": [ "src/auto-instrumentations/bundler/*.ts", "tests/**/*.{ts,tsx,mts,cts,cjs,mjs}",