多模态与附件

![Alt text](https://example.com/image.jpg)

{
  "content": [
    {
      "role": "system",
      "content": "You are an AI trained to describe and interpret images. Describe the main objects and actions in the image."
    },
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "What's happening in this image?"
        },
        {
          "type": "image_url",
          "image_url": {
            "url": "https://example.com/image.jpg"
          }
        }
      ]
    }
  ]
}

from langfuse import get_client, observe, propagate_attributes
from langfuse.media import LangfuseMedia

# Create a LangfuseMedia object from a file

with open("static/bitcoin.pdf", "rb") as pdf_file:
pdf_bytes = pdf_file.read()

# Wrap media in LangfuseMedia class

pdf_media = LangfuseMedia(content_bytes=pdf_bytes, content_type="application/pdf")

# Using with the decorator

@observe()
def process_document():
    langfuse = get_client()

    # Propagate metadata (including media) to all child observations
    with propagate_attributes(
        metadata={"document": pdf_media}
    ):
        pass

    # Or update the current span
    langfuse.update_current_span(
        input={"document": pdf_media}
    )

# Using with context managers

langfuse = get_client()

with langfuse.start_as_current_observation(as_type="span", name="analyze-document") as span: # Include media in the span input, output, or metadata
    span.update(
        input={"document": pdf_media},
        metadata={"file_size": len(pdf_bytes)}
    )

    # Process document...

    # Add results with media to the output
    span.update(output={
        "summary": "This document explains Bitcoin...",
        "original": pdf_media
    })

import fs from "fs";
import { LangfuseMedia } from "@langfuse/core";

// Wrap media in LangfuseMedia class
const wrappedMedia = new LangfuseMedia({
  source: "bytes",
  contentBytes: fs.readFileSync(new URL("./bitcoin.pdf", import.meta.url)),
  contentType: "application/pdf",
});

// Optionally, access media via wrappedMedia.obj
console.log(wrappedMedia.obj);

// Include media in any trace or observation
const span3 = startObservation("media-pdf-generation");

const generation3 = span3.startObservation('llm-call', {
  model: 'gpt-4',
  input: wrappedMedia,
}, {asType: "generation"});

generation3.end();

span3.end();

@@@langfuseMedia:type={MIME_TYPE}|id={LANGFUSE_MEDIA_ID}|source={SOURCE_TYPE}@@@

from langfuse import get_client

# Initialize Langfuse client
langfuse = get_client()

# Example object with media references
obj = {
    "image": "@@@langfuseMedia:type=image/jpeg|id=some-uuid|source=bytes@@@",
    "nested": {
        "pdf": "@@@langfuseMedia:type=application/pdf|id=some-other-uuid|source=bytes@@@"
    }
}

# Resolve media references to base64 data URIs
resolved_obj = langfuse.resolve_media_references(
    obj=obj,
    resolve_with="base64_data_uri"
)

# Result:
# {
#     "image": "data:image/jpeg;base64,/9j/4AAQSkZJRg...",
#     "nested": {
#         "pdf": "data:application/pdf;base64,JVBERi0xLjcK..."
#     }
# }

from langfuse import Langfuse

# Initialize Langfuse client
langfuse = Langfuse()

# Example object with media references
obj = {
    "image": "@@@langfuseMedia:type=image/jpeg|id=some-uuid|source=bytes@@@",
    "nested": {
        "pdf": "@@@langfuseMedia:type=application/pdf|id=some-other-uuid|source=bytes@@@"
    }
}

# Resolve media references to base64 data URIs
resolved_trace = langfuse.resolve_media_references(
    obj=obj,
    resolve_with="base64_data_uri"
)

# Result:
# {
#     "image": "data:image/jpeg;base64,/9j/4AAQSkZJRg...",
#     "nested": {
#         "pdf": "data:application/pdf;base64,JVBERi0xLjcK..."
#     }
# }

import { LangfuseClient } from "@langfuse/client";

const langfuse = new LangfuseClient()

// Example object with media references
const obj = {
  image: "@@@langfuseMedia:type=image/jpeg|id=some-uuid|source=bytes@@@",
  nested: {
    pdf: "@@@langfuseMedia:type=application/pdf|id=some-other-uuid|source=bytes@@@",
  },
};

// Resolve media references to base64 data URIs
const resolvedTrace = await langfuse.resolveMediaReferences({
  obj: obj,
  resolveWith: "base64DataUri",
});

// Result:
// {
//     image: "data:image/jpeg;base64,/9j/4AAQSkZJRg...",
//     nested: {
//         pdf: "data:application/pdf;base64,JVBERi0xLjcK..."
//     }
// }