Download notebook · View source · Open in Colab
Work with Descriptors (Embeddings)
A Descriptor stores an embedding: a numeric vector with optional metadata. A DescriptorSet groups embeddings in a common search space and defines their dimensions, index engine, and distance metric. This example stores a recipe embedding, searches for it, updates its metadata, and removes it.
The examples require Python 3.10 or later, an ApertureDB connection profile or environment credentials, and permission to create, read, update, and delete objects. A unique documentation_run property separates the example data from other records. Cleanup uses that value, so keep the setup variables available until cleanup completes.
Connect to ApertureDB
ApertureDB Cloud The SDK connects with an API key from your Cloud instance. Obtain the key from Connect > Generate API Key and add it to a .env file in this directory:
APERTUREDB_KEY=your_key_here
Local evaluation: follow the deployment guide, then configure your connection profile.
%pip install --quiet "aperturedb==0.5.1" "python-dotenv==1.1.1"
Note: you may need to restart the kernel to use updated packages.
# Option A: ApertureDB Cloud
from dotenv import load_dotenv
load_dotenv() # loads APERTUREDB_KEY from .env into the environment
False
# For local evaluation, configure the profile from your versioned deployment package.
# https://docs.aperturedata.io/administration/deployment/DockerCompose
from aperturedb.CommonLibrary import create_connector
client = create_connector()
import uuid
documentation_run = uuid.uuid4().hex
ingredient_class = "CookbookIngredient_" + documentation_run
connection_class = "CookbookIngredientLink_" + documentation_run
def checked_query(query, input_blobs=None):
response, output_blobs = client.query(query, input_blobs or [])
assert isinstance(response, list) and len(response) == len(query), response
for command in response:
result = next(iter(command.values()))
assert result.get("status") == 0, response
return response, output_blobs
response, _ = checked_query([{"GetStatus": {}}])
client.print_last_response()
[
{
"GetStatus": {
"info": "OK",
"status": 0,
"system": "ApertureDB",
"version": "0.19.15"
}
}
]
Define the search space for recipes
AddDescriptorSet creates a named search space for 1,024-dimensional recipe embeddings. This example selects the Flat engine and L2 distance metric.
import uuid
descriptorset_name = "recipe_search_" + uuid.uuid4().hex[:12]
q = [{
"AddDescriptorSet": { # a new descriptor set / collection is added only if the name doesn't exist
"name": descriptorset_name,
"dimensions": 1024,
"engine": "Flat", # It's possible to choose different engines or even multiple
"metric": "L2", # It's possible to choose different distance metrics or even multiple
"properties": {
"year_created": 2023,
"source": "ApertureDB cookbook dataset",
"model": "embed-english-v3.0",
"provider": "cohere"
}
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
[
{
"AddDescriptorSet": {
"status": 0
}
}
]
Prepare the embedding to be added
Embeddings can be loaded from a NumPy file or generated by an embedding model. The supplied file contains one 1,024-dimensional vector, which the example converts to float32 bytes for ingestion.
import numpy as np
# Download the sample file
from pathlib import Path
from urllib.request import urlopen
import hashlib
fixture_path = Path("data/embeddings.npy")
fixture_path.parent.mkdir(exist_ok=True)
fixture_url = "https://raw.githubusercontent.com/aperture-data/Cookbook/e333f6c59070b9165033d9ddd5af852a6b9624ba/notebooks/simple/data/embeddings.npy"
fixture_sha256 = "5ec479dd18d5a407c5081616279f4684a50910b3e8d51bd60b7cfdf827ec9d8f"
if not fixture_path.exists() or hashlib.sha256(fixture_path.read_bytes()).hexdigest() != fixture_sha256:
with urlopen(fixture_url, timeout=60) as response:
fixture_path.write_bytes(response.read())
assert hashlib.sha256(fixture_path.read_bytes()).hexdigest() == fixture_sha256
print("Verified fixture:", fixture_path.name)
# File can contain multiple descriptors of matching dimensions. So you can load them
# by passing the right index
embedding_npy_array = np.load("data/embeddings.npy")
index = 0 # We currently have only one embedding in the file
embedding = embedding_npy_array[index]
# expected byte array format for adding descriptor in ApertureDB
embedding_bytes = embedding.astype('float32').tobytes()
assert embedding.shape == (1024,)
assert np.isfinite(embedding).all()
Verified fixture: embeddings.npy
Add a Recipe Embedding for Similarity Search Later
AddDescriptor stores the vector in the named set with a label and metadata. The vector bytes accompany the command as an input blob.
The Python SDK loaders handle bulk ingestion.
q = [{
"AddDescriptor": {
"set": descriptorset_name, # descriptors / embeddings are always added to a set / collection
"label": "dinner",
"properties": {
'documentation_run': documentation_run,
"id": 75,
"year_created": 2023,
"source": "Cookbook recipe",
"model": "embed-english-v3.0",
"provider": "cohere"
},
"if_not_found": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]]
}
}]
responses, blobs = checked_query(q, [embedding_bytes])
print(client.get_last_response_str())
[
{
"AddDescriptor": {
"status": 0
}
}
]
K-NN Search For Matching Embedding
FindDescriptor searches the set using the same vector that was added. Because the set contains one descriptor, the response contains one match, including its label, properties, distance, and vector bytes.
q = [{
"FindDescriptor": {
'constraints': ['$documentation_run', '==', documentation_run],
# Specify the descriptor set in which to search.
"set": descriptorset_name,
"k_neighbors": 6,
"distances": True,
"labels": True,
"blobs": True,
"results": {
"all_properties": True
}
}
}]
responses, blobs = checked_query(q, [embedding_bytes])
print(client.get_last_response_str())
assert responses[0]["FindDescriptor"]["returned"] == 1
assert len(blobs) == 1
np.testing.assert_array_equal(np.frombuffer(blobs[0], dtype="float32"), embedding.astype("float32"))
[
{
"FindDescriptor": {
"blobs_start": 0,
"entities": [
{
"_blob_index": 0,
"_distance": 0.0,
"_label": "dinner",
"_set_name": "recipe_search_83e9fadd691d",
"_uniqueid": "3.0.540",
"documentation_run": "9d5b3a59886442e9a03a0dd23c8629d2",
"id": 75,
"model": "embed-english-v3.0",
"provider": "cohere",
"source": "Cookbook recipe",
"year_created": 2023
}
],
"returned": 1,
"status": 0
}
}
]
Remove Extra Properties with Update
UpdateDescriptor with remove_props removes the year_created property from the matching descriptor. The vector remains in the set.
q = [{
"UpdateDescriptor": {
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"remove_props": ["year_created"]
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
[
{
"UpdateDescriptor": {
"count": 1,
"status": 0
}
}
]
Read the updated descriptor
The descriptor is still returned, but its properties no longer include year_created.
q = [{
"FindDescriptor": {
# Specify the descriptor set in which to search.
"set": descriptorset_name,
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"results": {
"all_properties": True
}
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
assert responses[0]["FindDescriptor"]["returned"] == 1
assert "year_created" not in responses[0]["FindDescriptor"]["entities"][0]
[
{
"FindDescriptor": {
"entities": [
{
"_set_name": "recipe_search_83e9fadd691d",
"_uniqueid": "3.0.540",
"documentation_run": "9d5b3a59886442e9a03a0dd23c8629d2",
"id": 75,
"model": "embed-english-v3.0",
"provider": "cohere",
"source": "Cookbook recipe"
}
],
"returned": 1,
"status": 0
}
}
]
Remove the Embedding
DeleteDescriptor removes the descriptor selected by its metadata and this run's identifier.
q = [{
"DeleteDescriptor": {
"set": descriptorset_name,
"constraints": ['all', ['$provider', '==', 'cohere'], ['$documentation_run', '==', documentation_run]]
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
[
{
"DeleteDescriptor": {
"count": 1,
"status": 0
}
}
]
Verify Deletion
The same descriptor query returns no matches after deletion.
q = [{
"FindDescriptor": {
# Specify the descriptor set in which to search.
"set": descriptorset_name,
"constraints": ['all', ['$id', '==', 75], ['$documentation_run', '==', documentation_run]],
"results": {
"all_properties": True
}
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
assert responses[0]["FindDescriptor"]["returned"] == 0
[
{
"FindDescriptor": {
"returned": 0,
"status": 0
}
}
]
Remove the Descriptor Set
DeleteDescriptorSet removes the set created for this run.
q = [{
"DeleteDescriptorSet": {
"with_name": descriptorset_name
}
}]
responses, blobs = checked_query(q)
print(client.get_last_response_str())
remaining, _ = checked_query([{"FindDescriptorSet": {"with_name": descriptorset_name, "results": {"all_properties": True}}}])
assert remaining[0]["FindDescriptorSet"]["returned"] == 0
[
{
"DeleteDescriptorSet": {
"count": 1,
"status": 0
}
}
]
What's next?