API Reference

encode_image(images)

Encode images into embedding vectors.

from multimodal import VisionEncoder

encoder = VisionEncoder.from_pretrained("clip-vit-l14")

# Single image
embedding = encoder.encode_image(image)  # [1, 768]

# Batch of images
embeddings = encoder.encode_image(images)  # [B, 768]

# With preprocessing
embedding = encoder.encode_image(
    image,
    normalize=True,
    return_tensors="pt"
)

encode_text(texts)

Encode text strings into embedding vectors.

from multimodal import TextEncoder

encoder = TextEncoder.from_pretrained("clip-vit-l14")

# Single text
embedding = encoder.encode_text("a photo of a cat")  # [1, 768]

# Multiple texts
embeddings = encoder.encode_text([
    "a photo of a cat",
    "a photo of a dog"
])  # [2, 768]

# Class prompts for zero-shot
prompts = [f"a photo of a {c}" for c in classes]
text_features = encoder.encode_text(prompts)

tokenize(texts)

Tokenize text for manual processing.

tokens = encoder.tokenize(
    texts,
    max_length=77,
    padding="max_length",
    truncation=True
)
# Returns: {"input_ids": [B, 77], "attention_mask": [B, 77]}

transcribe(audio)

Transcribe audio to text with timestamps.

from multimodal import WhisperEncoder

whisper = WhisperEncoder.from_pretrained("whisper-medium")

# Basic transcription
result = whisper.transcribe(audio_path)
print(result["text"])

# With options
result = whisper.transcribe(
    audio_path,
    language="en",
    task="transcribe",  # or "translate"
    word_timestamps=True,
    beam_size=5
)

# Access word-level timestamps
for segment in result["segments"]:
    for word in segment["words"]:
        print(f"{word['text']}: {word['start']}-{word['end']}")

encode_audio(audio)

Get audio embeddings from encoder.

# Get encoder representations
embeddings = whisper.encode_audio(audio)  # [B, T, D]

# Pool to single vector
pooled = embeddings.mean(dim=1)  # [B, D]

compute_similarity(a, b)

from multimodal import compute_similarity

# Image-text similarity
similarity = compute_similarity(
    image_features,  # [N, D]
    text_features,   # [M, D]
    mode="cosine"    # or "dot"
)  # [N, M]

# Get predictions
probs = similarity.softmax(dim=-1)

ImageTextRetriever

from multimodal import ImageTextRetriever

retriever = ImageTextRetriever(
    vision_encoder="clip-vit-l14",
    text_encoder="clip-vit-l14",
    index_type="faiss"  # or "exact"
)

# Build index from images
retriever.index_images(image_paths)

# Search with text query
results = retriever.search(
    query="a sunset over the ocean",
    top_k=10
)

for r in results:
    print(f"{r['path']}: {r['score']:.3f}")