sdk-multimodal

Multimodal messages through the SDK — images and text as first-class content.

Run it from sema/:

sema check examples/sdk-multimodal
SEMA_STRICT=1 sema run examples/sdk-multimodal
sema assure examples/sdk-multimodal --grade silver

Source

src/main.sema

# End-to-end multimodal via the Sema SDK: real small HF models behind native
# Sema functions. Runs for real once `sema get-models` extras are installed.
from sdk_multimodal.ai import caption, ocr, vqa, transcribe, speak

def main() -> None !{proc.run, fs.read, fs.write, observe.record}:
    # Vision: describe an image; OCR: read the printed text; VQA: answer about it.
    log.info("caption", text=caption("text.png"))
    log.info("ocr", text=ocr("text.png"))
    log.info("vqa", answer=vqa("text.png", "what color is the box?"))
    # Speech round-trip: TTS writes audio, STT reads it back.
    speak("the quick brown fox jumps over the lazy dog", "spoken.wav")
    log.info("stt (round-trip)", text=transcribe("spoken.wav"))

src/ai.sema

# Sema SDK — the multimodal capability surface, written in Sema.
#
# The language does not ship blank: this module is the "standard AI library".
# Each function is a clean native Sema interface; the backend is chosen by the
# config/model registry (§5.38/§5.43). Text + embeddings run on the built-in /
# GGUF engines out of the box; vision/OCR/STT/TTS reuse small HuggingFace models
# through the Python bridge (§5.44) — we don't reinvent well-solved wheels, we
# bind them behind a Sema interface. Swap any backend in `sema.toml` (D48/D51).

import python
import tools

# ---- text ----------------------------------------------------------------

def chat(prompt: str) -> str !{model.invoke}:
    sem "Generate a natural-language response with the configured text model"
    return tools.run(prompt, []).get("answer")

def ask(question: str, toolset: list) -> str !{model.invoke}:
    sem "Answer a question, letting the model call the given Sema functions as tools"
    return tools.run(question, toolset).get("answer")

# ---- embeddings ----------------------------------------------------------

def embed_text(text: str) -> list[f64] !{model.embed}:
    sem "Embed text into a vector with the configured embedding model"
    return embed(text)

def similarity(a: str, b: str) -> f64 !{model.embed}:
    sem "Cosine similarity of two texts' embeddings"
    return (a ~= b).score

# ---- vision (reuses a small HF caption/vision model) ---------------------

def caption(image_path: str) -> str !{proc.run, fs.read}:
    sem "Describe an image in natural language"
    return python.call("sema_sdk.vision", "caption", [image_path])

def vqa(image_path: str, question: str) -> str !{proc.run, fs.read}:
    sem "Answer a question about an image"
    return python.call("sema_sdk.vision", "vqa", [image_path, question])

# ---- OCR -----------------------------------------------------------------

def ocr(image_path: str) -> str !{proc.run, fs.read}:
    sem "Extract text from an image (OCR)"
    return python.call("sema_sdk.ocr", "read", [image_path])

# ---- speech --------------------------------------------------------------

def transcribe(audio_path: str) -> str !{proc.run, fs.read}:
    sem "Transcribe speech from an audio file to text (STT)"
    return python.call("sema_sdk.stt", "transcribe", [audio_path])

def speak(text: str, out_path: str) -> str !{proc.run, fs.write}:
    sem "Synthesize speech audio from text (TTS); returns the output path"
    return python.call("sema_sdk.tts", "speak", [text, out_path])

Reflected API

ai

def chat

def chat(prompt: str) -> str !{model.invoke}

Parameters

name	type
prompt	str

Returns str

Effects !{model.invoke}

def ask

def ask(question: str, toolset: list) -> str !{model.invoke}

Parameters

name	type
question	str
toolset	list

Returns str

Effects !{model.invoke}

def embed_text

def embed_text(text: str) -> list[f64] !{model.embed}

Parameters

name	type
text	str

Returns list[f64]

Effects !{model.embed}

def similarity

def similarity(a: str, b: str) -> f64 !{model.embed}

Parameters

name	type
a	str
b	str

Returns f64

Effects !{model.embed}

def caption

def caption(image_path: str) -> str !{proc.run, fs.read}

Parameters

name	type
image_path	str

Returns str

Effects !{proc.run, fs.read}

def vqa

def vqa(image_path: str, question: str) -> str !{proc.run, fs.read}

Parameters

name	type
image_path	str
question	str

Returns str

Effects !{proc.run, fs.read}

def ocr

def ocr(image_path: str) -> str !{proc.run, fs.read}

Parameters

name	type
image_path	str

Returns str

Effects !{proc.run, fs.read}

def transcribe

def transcribe(audio_path: str) -> str !{proc.run, fs.read}

Parameters

name	type
audio_path	str

Returns str

Effects !{proc.run, fs.read}

def speak

def speak(text: str, out_path: str) -> str !{proc.run, fs.write}

Parameters

name	type
text	str
out_path	str

Returns str

Effects !{proc.run, fs.write}

main

def main

def main() -> None !{proc.run, fs.read, fs.write, observe.record}

Returns None

Effects !{proc.run, fs.read, fs.write, observe.record}