Download notebook · View source · Open in Colab

Image Vector Search

CLIP maps images and text to the same vector space. ApertureDB stores the image embeddings as descriptors linked to their source images, so a text query can retrieve matching photographs. This example uses food images from the Cookbook dataset.

Connect to ApertureDB

ApertureDB Cloud The SDK connects with an API key from your Cloud instance. Obtain the key from Connect > Generate API Key and add it to a .env file in this directory:

APERTUREDB_KEY=your_key_here

Local evaluation: follow the deployment guide, then configure your connection profile.

See client configuration options for all connection methods and server setup options for deployment choices.

%pip install --quiet "aperturedb==0.5.1" "python-dotenv==1.1.1" sentence-transformers Pillow requests pandas

from dotenv import load_dotenv
load_dotenv()

# For local evaluation, configure the profile from your versioned deployment package.
# https://docs.aperturedata.io/administration/deployment/DockerCompose

from aperturedb.CommonLibrary import create_connector

client = create_connector()
response, _ = client.query([{"GetStatus": {}}])
client.print_last_response()

Step: Load the CLIP model

The same CLIP model encodes both the stored images and the search text.

from sentence_transformers import SentenceTransformer
model = SentenceTransformer("clip-ViT-B-32")
print(f"Embedding dimensions: {model.get_sentence_embedding_dimension()}")

Step: Load image URLs from Cookbook dataset

The Cookbook data supplies image URLs and metadata describing each dish.

import pandas as pd
dishes = pd.read_csv(
    "https://raw.githubusercontent.com/aperture-data/Cookbook/51b3c3d9f495f8fd77108a270b88fe8ab8cc0004/images.adb.csv"
)
dishes = dishes.head(10)   # use first 10 for this demo
print(f"Loaded {len(dishes)} dishes")
print(dishes[["dish_name", "url", "food_tags"]].to_string(index=False))

Step: Create a DescriptorSet for image embeddings

The descriptor set stores the CLIP vectors with the dimensions and metric expected by the model.

import uuid
SET_NAME = "dish_image_search_" + uuid.uuid4().hex[:12]

client.query([{
    "AddDescriptorSet": {
        "name":       SET_NAME,
        "dimensions": 512,     # CLIP ViT-B/32 output size
        "engine":     "HNSW",  # approximate nearest neighbor for larger sets
        "metric":     "CS",    # cosine similarity
    }
}])
client.print_last_response()

Step: Add images with embeddings in one transaction

Each query contains AddImage and AddDescriptor. The connect parameter links the descriptor to the image within that transaction. CLIP encodes the source image before its vector is sent to ApertureDB.

import requests
from PIL import Image
from io import BytesIO
import numpy as np

def fix_url(url):
    """Pin Cookbook image URLs and normalize spaces in fixture filenames."""
    return url.replace("refs/heads/main", "51b3c3d9f495f8fd77108a270b88fe8ab8cc0004").replace("/main/", "/51b3c3d9f495f8fd77108a270b88fe8ab8cc0004/").replace("%20", "_").replace(" ", "_")

for _, row in dishes.iterrows():
    img_url = fix_url(row["url"])
    resp = requests.get(img_url, timeout=10)
    resp.raise_for_status()
    img_bytes = resp.content
    img = Image.open(BytesIO(img_bytes)).convert("RGB")
    emb = model.encode(img, normalize_embeddings=True).astype("float32")

    q = [
        {
            "AddImage": {
                "_ref": 1,
                "properties": {
                    "dish_name": row["dish_name"],
                    "cuisine":   row["food_tags"],
                    "documentation_run": SET_NAME,
                }
            }
        },
        {
            "AddDescriptor": {
                "set":   SET_NAME,
                "properties": {
                    "dish_name": row["dish_name"],
                    "cuisine":   row["food_tags"],
                    "documentation_run": SET_NAME,
                },
                "connect": {
                    "ref": 1,
                    "class": "has_embedding"
                }
            }
        }
    ]
    # blob[0] -> AddImage (raw image bytes), blob[1] -> AddDescriptor (embedding)
    client.query(q, [img_bytes, emb.tobytes()])

print("Done adding images and embeddings")

Step: Text-to-image search

CLIP encodes the text query in the same space as the image embeddings. Vector search selects matching descriptors, and their connections identify the source images.

query_text = "creamy curry with rice"

query_emb = model.encode(query_text, normalize_embeddings=True).astype("float32")

# Find top-3 descriptors, then traverse to linked images
q = [
    {
        "FindDescriptor": {
            "set":         SET_NAME,
            "k_neighbors": 3,
            "distances":   True,
            "_ref":        1,
            "uniqueids":   True,
            "results":     {"all_properties": True},
        }
    },
    {
        "FindImage": {
            "is_connected_to": {"ref": 1, "connection_class": "has_embedding"},
            "blobs":           False,
            "group_by_source": True,
            "results":         {"all_properties": True},
        }
    }
]

response, _ = client.query(q, [query_emb.tobytes()])

if not isinstance(response, list) or any(next(iter(item.values())).get("status") != 0 for item in response):
    print("Query failed — check the response:")
    client.print_last_response()
else:
    descriptors = response[0]["FindDescriptor"].get("entities", [])
    images_by_descriptor = response[1]["FindImage"].get("entities", {})

    print(f'Query: "{query_text}"\n')
    for desc in descriptors:
        for img in images_by_descriptor.get(desc["_uniqueid"], []):
            print(f"  {img['dish_name']:<25} cuisine={img['cuisine']:<15} distance={desc['_distance']:.4f}")

Step: Cleanup (optional)

client.query([{"DeleteDescriptorSet": {"with_name": SET_NAME}}])
client.print_last_response()
client.query([{"DeleteImage": {"constraints": ["$documentation_run", "==", SET_NAME]}}])
client.print_last_response()