Download notebook · View source · Open in Colab
BoundingBox and Polygon Annotations on Images
A BoundingBox describes a rectangle within an image; a Polygon describes a region by its vertices. Both store annotations and refer to their source image. Queries can return region geometry, the corresponding pixels, or images selected through their annotations.
Connect to the database
The examples require a running database and configured client credentials. The quick start guide describes the setup.
# Install the required client packages if needed
%pip install --quiet "aperturedb==0.5.1"
Note: you may need to restart the kernel to use updated packages.
from aperturedb.CommonLibrary import create_connector
# Create the connector for ApertureDB
client = create_connector()
def checked_query(query, input_blobs=None):
response, output_blobs = client.query(query, input_blobs or [])
assert isinstance(response, list) and len(response) == len(query), response
for command in response:
assert next(iter(command.values())).get("status") == 0, response
return response, output_blobs
import uuid
documentation_run = uuid.uuid4().hex
# Simple query to see how the database is doing
# https://docs.aperturedata.io/query_language/Reference/db_commands/GetStatus
query = [{
"GetStatus": {
}
}]
# Execute the query to get back a JSON response for GetStatus
response, blobs = checked_query(query)
client.print_last_response()
[
{
"GetStatus": {
"info": "OK",
"status": 0,
"system": "ApertureDB",
"version": "0.19.15"
}
}
]
Add an Image of a Dish and Annotate it
Regions of interest, including bounding boxes and polygons, belong to an image. A query can add the image and its regions together, or attach regions to an existing image.
Annotations can come from labeling integrations such as Label Studio. This example defines a bounding box around a pan and a polygon around the bread in the supplied photograph.
from pathlib import Path
from urllib.request import urlopen
import hashlib
fixtures = {'focaccia_brittany.jpg': '4b104ba601ab3bf1126579e9dc795842551fc25be9ab2640795da2fb461f7061'}
for filename, expected_sha256 in fixtures.items():
path = Path("data") / filename
path.parent.mkdir(exist_ok=True)
url = "https://raw.githubusercontent.com/aperture-data/Cookbook/e333f6c59070b9165033d9ddd5af852a6b9624ba/notebooks/simple/data/" + filename
if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_sha256:
with urlopen(url, timeout=60) as response:
path.write_bytes(response.read())
assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256
print("Verified fixture:", filename)
Verified fixture: focaccia_brittany.jpg
query = [{
"AddImage": {
"_ref": 1, # Assign a reference to the result of this query which is this image we are adding
"properties": {'name': 'Focaccia', 'id': 35, 'contributor': 'Vishakha', 'category': 'starter', 'cuisine': 'Italian', 'location': 'Brittany', 'caption': 'fresh made focaccia with herbs from the garden', 'documentation_run': documentation_run},
"if_not_found": ['all', ['$id', '==', 35], ['$documentation_run', '==', documentation_run]]
}
}, {
"AddBoundingBox": {
"image_ref": 1, # Use the reference attached in the previous command to indicate the source of this bounding box
"rectangle": {
"x": 40, # pixels offset from the left edge
"y": 1500, # pixels offset from the top edge
"width": 2200, # box width in pixels
"height": 1800 # box height in pixels
},
"label": "Pan",
"properties": {'name': 'iron pan', 'annotation_source': 'manual', 'documentation_run': documentation_run},
"if_not_found": ['all', ['all', ['$name', '==', 'iron pan'], ['$annotation_source', '==', 'manual']], ['$documentation_run', '==', documentation_run]]
}
}, {
"AddPolygon": {
"image_ref": 1, # Use the reference attached in the previous command to indicate the source of this bounding box
"polygons": [
[
[483.84,2890.94],
[416.40,2314.37],
[704.592,1935.36],
[1391.04,1854.72],
[1814.4,2177.28],
[1844.64,2741.76],
[1118.88,3104.64]
]
],
"label": "Bread",
"properties": {'name': 'Homemade Foccacia', 'annotation_source': 'labelstudio', 'documentation_run': documentation_run},
"if_not_found": ['all', ['all', ['$name', '==', 'Homemade Foccacia'], ['$annotation_source', '==', 'labelstudio']], ['$documentation_run', '==', documentation_run]]
}
}]
# Read the image data as a binary blob
fd = open("data/focaccia_brittany.jpg", 'rb')
image_arr = [ fd.read() ]
fd.close()
response, blobs = checked_query(query, image_arr)
client.print_last_response()
bread_vertices = query[2]["AddPolygon"]["polygons"][0]
[
{
"AddImage": {
"status": 0
}
},
{
"AddBoundingBox": {
"status": 0
}
},
{
"AddPolygon": {
"status": 0
}
}
]
Retrieve Pixels in Regions of Interest
The query finds the image, then selects its Pan bounding box and Bread polygon. With blobs: True, the region commands return their image pixels alongside the requested geometry.
from IPython.display import Image, display
query = [ {
"FindImage": {
"_ref": 1,
"constraints": ['all', ['all', ['$name', '==', 'Focaccia'], ['$location', '>', 'A']], ['$documentation_run', '==', documentation_run]],
"blobs": False # Don't retrieve pixel data from the image for this command...
}
}, {
"FindBoundingBox": {
'constraints': ['$documentation_run', '==', documentation_run],
"is_connected_to": {
"ref": 1
},
"with_label": "Pan",
"results": {
"all_properties": True
},
"coordinates": True,
"blobs": True # Instead, retrieve pixel data from the bounding box...
}
}, {
"FindPolygon": {
'constraints': ['$documentation_run', '==', documentation_run],
"is_connected_to": {
"ref": 1
},
"with_label": "Bread",
"results": {
"all_properties": True
},
"vertices": True,
"blobs": True # ... and in this polygon
}
}
]
res, blobs = checked_query(query)
client.print_last_response()
from IPython.display import display,Image
image = Image(data=blobs[0])
display(image)
image = Image(data=blobs[1])
display(image)
from io import BytesIO
from PIL import Image as PILImage
assert res[1]["FindBoundingBox"]["returned"] == 1
assert res[2]["FindPolygon"]["returned"] == 1
assert len(blobs) == 2
assert PILImage.open(BytesIO(blobs[0])).size == (2200, 1800)
assert all(PILImage.open(BytesIO(blob)).width > 0 for blob in blobs)
[
{
"FindImage": {
"returned": 0,
"status": 0
}
},
{
"FindBoundingBox": {
"blobs_start": 0,
"entities": [
{
"_area": 3960000,
"_blob_index": 0,
"_coordinates": {
"height": 1800,
"width": 2200,
"x": 40,
"y": 1500
},
"_label": "Pan",
"_uniqueid": "1.0.180",
"annotation_source": "manual",
"documentation_run": "e21aa616b6bf42028c92fd5ded16bf5c",
"name": "iron pan"
}
],
"returned": 1,
"status": 0
}
},
{
"FindPolygon": {
"blobs_start": 1,
"entities": [
{
"_area": 1359696,
"_blob_index": 1,
"_label": "Bread",
"_uniqueid": "4.0.180",
"_vertices": [
[
[
483.8399963378906,
2890.93994140625
],
[
416.3999938964844,
2314.3701171875
],
[
704.5919799804688,
1935.3599853515625
],
[
1391.0400390625,
1854.719970703125
],
[
1814.4000244140625,
2177.280029296875
],
[
1844.6400146484375,
2741.760009765625
],
[
1118.8800048828125,
3104.639892578125
]
]
],
"annotation_source": "labelstudio",
"documentation_run": "e21aa616b6bf42028c92fd5ded16bf5c",
"name": "Homemade Foccacia"
}
],
"returned": 1,
"status": 0
}
}
]
Find an image by its label
The query finds a bounding box labeled Pan, then follows its connection to return the source image.
query = [ {
"FindBoundingBox": {
'constraints': ['$documentation_run', '==', documentation_run],
"_ref": 1,
"with_label": "Pan",
"coordinates": True,
"labels": True,
"blobs": False
}}, {
"FindImage": {
'constraints': ['$documentation_run', '==', documentation_run],
"is_connected_to": {
"ref": 1
},
"blobs": True
}
} ]
res, blobs = checked_query(query)
client.print_last_response()
for image_blob in blobs:
display(Image(data=image_blob))
assert res[0]["FindBoundingBox"]["returned"] == 1
assert res[1]["FindImage"]["returned"] == 1
assert len(blobs) == 1
[
{
"FindBoundingBox": {
"entities": [
{
"_coordinates": {
"height": 1800,
"width": 2200,
"x": 40,
"y": 1500
},
"_label": "Pan"
}
],
"returned": 1,
"status": 0
}
},
{
"FindImage": {
"blobs_start": 0,
"returned": 1,
"status": 0
}
}
]
Find Polygon pixels by label
The query finds the Bread polygon, requests its geometry, and follows its connection to retrieve the source image.
query = [ {
"FindPolygon": {
'constraints': ['$documentation_run', '==', documentation_run],
"_ref": 1,
"with_label": "Bread",
"vertices": True,
"areas": True,
"hulls": True,
"bounds": True,
"labels": True,
"blobs": False
}}, {
"FindImage": {
'constraints': ['$documentation_run', '==', documentation_run],
"is_connected_to": {
"ref": 1
},
"blobs": True
}
} ]
res, blobs = checked_query(query)
client.print_last_response()
for image_blob in blobs:
display(Image(data=image_blob))
assert res[0]["FindPolygon"]["returned"] == 1
assert res[1]["FindImage"]["returned"] == 1
assert len(blobs) == 1
[
{
"FindPolygon": {
"entities": [
{
"_area": 1359696,
"_bounds": {
"height": 1251,
"width": 1429,
"x": 416,
"y": 1854
},
"_hulls": [
[
[
1391.0400390625,
1854.719970703125
],
[
1814.4000244140625,
2177.280029296875
],
[
1844.6400146484375,
2741.760009765625
],
[
1118.8800048828125,
3104.639892578125
],
[
483.8399963378906,
2890.93994140625
],
[
416.3999938964844,
2314.3701171875
],
[
704.5919799804688,
1935.3599853515625
]
]
],
"_label": "Bread",
"_vertices": [
[
[
483.8399963378906,
2890.93994140625
],
[
416.3999938964844,
2314.3701171875
],
[
704.5919799804688,
1935.3599853515625
],
[
1391.0400390625,
1854.719970703125
],
[
1814.4000244140625,
2177.280029296875
],
[
1844.6400146484375,
2741.760009765625
],
[
1118.8800048828125,
3104.639892578125
]
]
]
}
],
"returned": 1,
"status": 0
}
},
{
"FindImage": {
"blobs_start": 0,
"returned": 1,
"status": 0
}
}
]
Compute IoU
RegionIoU measures overlap as the intersection area divided by the union area. This query compares the pan bounding box with the bread polygon.
query = [ {
"FindBoundingBox": {
'constraints': ['$documentation_run', '==', documentation_run],
"_ref": 1,
"with_label": "Pan",
"coordinates": True,
"labels": True,
"blobs": False
}
},{
"FindPolygon": {
'constraints': ['$documentation_run', '==', documentation_run],
"_ref": 2,
"with_label": "Bread",
"bounds": True,
"labels": True,
"blobs": False
}
},{
"RegionIoU": {
"roi_1": 1,
"roi_2": 2
}
}]
res, blobs = checked_query(query)
client.print_last_response()
import math
# Every bread vertex is inside the pan's bounding box, so union area is the box area.
polygon_area = abs(sum(a[0] * b[1] - b[0] * a[1] for a, b in zip(bread_vertices, bread_vertices[1:] + bread_vertices[:1]))) / 2
expected_iou = polygon_area / (2200 * 1800)
assert len(res[2]["RegionIoU"]["IoU"]) == 1
assert len(res[2]["RegionIoU"]["IoU"][0]) == 1
assert math.isclose(res[2]["RegionIoU"]["IoU"][0][0], expected_iou, rel_tol=1e-5)
print("Verified IoU:", res[2]["RegionIoU"]["IoU"][0][0])
[
{
"FindBoundingBox": {
"entities": [
{
"_coordinates": {
"height": 1800,
"width": 2200,
"x": 40,
"y": 1500
},
"_label": "Pan"
}
],
"returned": 1,
"status": 0
}
},
{
"FindPolygon": {
"entities": [
{
"_bounds": {
"height": 1251,
"width": 1429,
"x": 416,
"y": 1854
},
"_label": "Bread"
}
],
"returned": 1,
"status": 0
}
},
{
"RegionIoU": {
"IoU": [
[
0.34335757575757575
]
],
"status": 0
}
}
]
Verified IoU: 0.34335757575757575
Delete the regions of interest and cleanup.
Cleanup removes the bounding box and polygon before deleting this run's source image.
query = [ {
"DeleteBoundingBox": { # To delete by label, do a Find with a ref and then delete using the ref
"constraints": ['all', ['$annotation_source', '==', 'manual'], ['$documentation_run', '==', documentation_run]]
}
},{
"DeletePolygon": {
"constraints": ['all', ['$annotation_source', '==', 'labelstudio'], ['$documentation_run', '==', documentation_run]]
}
},{
"DeleteImage": {
"constraints": ['all', ['$name', '==', 'Focaccia'], ['$documentation_run', '==', documentation_run]]
}
}]
res, blobs = checked_query(query)
client.print_last_response()
# Verify cleanup for every object type this run created.
remaining, _ = checked_query([
{"FindBoundingBox": {"constraints": ["$documentation_run", "==", documentation_run], "results": {"all_properties": True}}},
{"FindPolygon": {"constraints": ["$documentation_run", "==", documentation_run], "results": {"all_properties": True}}},
{"FindImage": {"constraints": ["$documentation_run", "==", documentation_run], "results": {"all_properties": True}}},
])
assert all(next(iter(command.values()))["returned"] == 0 for command in remaining)
[
{
"DeleteBoundingBox": {
"count": 1,
"status": 0
}
},
{
"DeletePolygon": {
"count": 1,
"status": 0
}
},
{
"DeleteImage": {
"count": 1,
"status": 0
}
}
]
What's next?
- Bulk load bounding boxes and polygons
- Add application-defined entities or try intersection over union to ApertureDB