Download notebook · View source · Open in Colab

Image and Video Preprocessing or Augmentation

ApertureDB applies preprocessing and augmentation operations when storing or retrieving images, videos, and regions of interest. Operations on retrieval produce a transformed result without requiring a separately stored copy. The examples convert an image format, retrieve a thumbnail, and sample and resize a video.

The examples require Python 3.10 or later, a configured ApertureDB connection, and permission to create, read, and delete objects. The sample media comes from a fixed repository revision; SHA-256 checks identify incomplete or changed downloads. A unique documentation_run property limits queries and cleanup to this run. Keep the setup variables available until cleanup completes.

# Install the required client packages if needed
%pip install --quiet "aperturedb==0.5.1"

Note: you may need to restart the kernel to use updated packages.

from aperturedb.CommonLibrary import create_connector

# Create the connector for ApertureDB
client = create_connector()

def checked_query(query, input_blobs=None):
    response, output_blobs = client.query(query, input_blobs or [])
    assert isinstance(response, list) and len(response) == len(query), response
    for command in response:
        assert next(iter(command.values())).get("status") == 0, response
    return response, output_blobs

import uuid
documentation_run = uuid.uuid4().hex

# Simple query to see how the database is doing
# https://docs.aperturedata.io/query_language/Reference/db_commands/GetStatus
query = [{
    "GetStatus": {
    }
}]

# Execute the query to get back a JSON response for GetStatus 
response, blobs = checked_query(query)

client.print_last_response()

[
    {
        "GetStatus": {
            "info": "OK",
            "status": 0,
            "system": "ApertureDB",
            "version": "0.19.15"
        }
    }
]

Add a reformatted Image of a Dish to ApertureDB

The source image is JPEG. Setting format to png in AddImage stores it as PNG.

from pathlib import Path
from urllib.request import urlopen
import hashlib

fixtures = {'focaccia_brittany.jpg': '4b104ba601ab3bf1126579e9dc795842551fc25be9ab2640795da2fb461f7061', 'crepe_flambe.mp4': '53275e11da0dcef74044be4ec9fd219d56e83fc8f57ae900da0120bf8551eaf1'}
for filename, expected_sha256 in fixtures.items():
    path = Path("data") / filename
    path.parent.mkdir(exist_ok=True)
    url = "https://raw.githubusercontent.com/aperture-data/Cookbook/e333f6c59070b9165033d9ddd5af852a6b9624ba/notebooks/simple/data/" + filename
    if not path.exists() or hashlib.sha256(path.read_bytes()).hexdigest() != expected_sha256:
        with urlopen(url, timeout=60) as response:
            path.write_bytes(response.read())
    assert hashlib.sha256(path.read_bytes()).hexdigest() == expected_sha256
    print("Verified fixture:", filename)

Verified fixture: focaccia_brittany.jpg

Verified fixture: crepe_flambe.mp4

query = [{
    "AddImage": {
        # Notice the missing "class" property since we already know its an Image (represented as _Image in ApertureDB)
        "properties": {
            'documentation_run': documentation_run,
            "name": "Focaccia",
            "id": 35,
            "contributor": "Vishakha",
            "category": "starter",
            "cuisine": "Italian",
            "location": "Brittany",
            "caption": "fresh made focaccia with herbs from the garden"
        },
        "format": "png",
        "if_not_found": ['all', ['$id', '==', 35], ['$documentation_run', '==', documentation_run]]
    }
}]

# Read the image data as a binary blob
fd = open("data/focaccia_brittany.jpg", 'rb')
image_arr = [ fd.read() ]
fd.close()

response, blobs = checked_query(query, image_arr)

client.print_last_response()

from io import BytesIO
from PIL import Image as PILImage
stored, stored_blobs = checked_query([{"FindImage": {"constraints": ["$documentation_run", "==", documentation_run], "blobs": True}}])
assert stored[0]["FindImage"]["returned"] == 1
assert PILImage.open(BytesIO(stored_blobs[0])).format == "PNG"

[
    {
        "AddImage": {
            "status": 0
        }
    }
]

Retrieve an image thumbnail

The query resizes the stored image to 100 × 100 pixels and returns JPEG bytes. The stored PNG remains available for other retrieval operations.

from aperturedb import NotebookHelpers as nh   # Our helper package for image displays and other utilities

query = [{
    "FindImage": {
        "constraints": ['all', ['all', ['$name', '==', 'Focaccia'], ['$location', '>', 'A']], ['$documentation_run', '==', documentation_run]],
        "operations": [{
            "type": "resize",
            "width": 100,
            "height": 100
        } ],
        "as_format": "jpg",
        "blobs": True
    }
}]

response, blobs = checked_query(query)

# Instead of using nh.display, which automatically reformats an image
# for display, we use IPython.display to show how its returned from the query.
from IPython.display import display,Image
num_images = response[0]["FindImage"]["returned"]
print(num_images)
for count in range(num_images):
    image = Image(blobs[count], format="JPEG")
    display(image)

assert num_images == 1 and len(blobs) == 1
thumbnail = PILImage.open(BytesIO(blobs[0]))
assert thumbnail.size == (100, 100) and thumbnail.format == "JPEG"

jpeg

Videos can similarly be pre-processed when adding or searching

Video commands also accept operations during ingestion or retrieval. This query stores the original cooking video so the following queries can return either the original or a transformed result.

query = [{
    "AddVideo": {
        # Notice the missing "class" property since we already know its a Video (represented as _Video in ApertureDB)
        "properties": {
            'documentation_run': documentation_run,
            "name": "crepe_flambe",
            "id": 45,
            "category": "dessert",
            "cuisine": "French",
            "location": "Brittany",
            "caption": "Special Brittany flambe crepe"
        },
        "if_not_found": ['all', ['$id', '==', 45], ['$documentation_run', '==', documentation_run]]
    }
}]

# Read the image data as a binary blob
fd = open("data/crepe_flambe.mp4", 'rb')
array = [ fd.read() ]
fd.close()

response, blobs = checked_query(query, array)

client.print_last_response()

[
    {
        "AddVideo": {
            "status": 0
        }
    }
]

Original video

FindVideo with blobs: True returns the stored video bytes.

from aperturedb import NotebookHelpers as nh   # Our helper package for image displays and other utilities

query = [{
    "FindVideo": {
        "constraints": ['all', ['all', ['$name', '>=', 'crepe'], ['$location', '>', 'A']], ['$documentation_run', '==', documentation_run]],
        "blobs": True,                          # This is set to False by default
        "results": {
            "all_properties": True
        }
    }
}]

response, blobs = checked_query(query)

client.print_last_response()
num_videos = response[0]["FindVideo"]["returned"]
for count in range(num_videos):
    nh.display_video_mp4(blobs[count])


import cv2
import tempfile
from pathlib import Path

def video_info(video_blob):
    with tempfile.TemporaryDirectory() as temporary:
        path = Path(temporary) / "video.mp4"
        path.write_bytes(video_blob)
        capture = cv2.VideoCapture(str(path))
        try:
            assert capture.isOpened(), "Returned video could not be decoded"
            return {
                "width": int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)),
                "height": int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)),
                "frames": int(capture.get(cv2.CAP_PROP_FRAME_COUNT)),
            }
        finally:
            capture.release()

assert num_videos == 1 and len(blobs) == 1
original_video = video_info(blobs[0])
assert original_video["frames"] > 150

[
    {
        "FindVideo": {
            "blobs_start": 0,
            "entities": [
                {
                    "_blob_index": 0,
                    "_duration_us": 5500000,
                    "_fps": 30,
                    "_frame_count": 165,
                    "_frame_height": 480,
                    "_frame_width": 852,
                    "_uniqueid": "5.0.100",
                    "caption": "Special Brittany flambe crepe",
                    "category": "dessert",
                    "cuisine": "French",
                    "documentation_run": "beac3ae3ddfc445da883bb8c17e4248f",
                    "id": 45,
                    "location": "Brittany",
                    "name": "crepe_flambe"
                }
            ],
            "returned": 1,
            "status": 0
        }
    }
]

Sample and pre-process the cooking video when retrieving

The query resizes the video to 300 × 300 pixels and returns every fifth frame from frame 30 through frame 150. The result contains 25 frames.

query = [{
    "FindVideo": {
        "constraints": ['all', ['all', ['$name', '>=', 'crepe'], ['$location', '>', 'A']], ['$documentation_run', '==', documentation_run]],
        "blobs": True,
        "operations": [{        # Original videos are longer. We just want to sample and prepare them
            "type": "resize",   # performance depends on the video complexity
            "height": 300,
            "width":  300
        },{
            "type": "interval", # Interval Operation, performance depends on the video complexity
            "start": 30,        # Start at frame 30 (frame numbering begins at zero)
            "stop": 150,        # Include frame 150
            "step": 5           # Read every 5th frame
        }]
    }
}]

response, blobs = checked_query(query)

print(client.get_last_response_str())
nh.display_video_mp4(blobs[0])

assert response[0]["FindVideo"]["returned"] == 1 and len(blobs) == 1
transformed_video = video_info(blobs[0])
assert (transformed_video["width"], transformed_video["height"]) == (300, 300)
assert transformed_video["frames"] == len(range(30, 151, 5)), transformed_video
print("Transformed video:", transformed_video)

[
    {
        "FindVideo": {
            "blobs_start": 0,
            "returned": 1,
            "status": 0
        }
    }
]

Transformed video: {'width': 300, 'height': 300, 'frames': 25}

Cleanup

The cleanup queries remove only the image and video marked with this run's identifier.

query = [{
    "DeleteVideo": {
        "constraints": ['all', ['$name', '==', 'crepe_flambe'], ['$documentation_run', '==', documentation_run]]
    }
},{
    "DeleteImage": {
        "constraints": ['all', ['$name', '==', 'Focaccia'], ['$documentation_run', '==', documentation_run]]
    }
}]

response, blobs = checked_query(query)

client.print_last_response()
# Verify cleanup for every object type this run created.
remaining, _ = checked_query([
    {"FindImage": {"constraints": ["$documentation_run", "==", documentation_run], "results": {"all_properties": True}}},
    {"FindVideo": {"constraints": ["$documentation_run", "==", documentation_run], "results": {"all_properties": True}}},
])
assert all(next(iter(command.values()))["returned"] == 0 for command in remaining)

[
    {
        "DeleteVideo": {
            "count": 1,
            "status": 0
        }
    },
    {
        "DeleteImage": {
            "count": 1,
            "status": 0
        }
    }
]

What's next?

Prepare datasets for ML
Apply operations on clips or frames on videos

Add a reformatted Image of a Dish to ApertureDB​

Retrieve an image thumbnail​

Videos can similarly be pre-processed when adding or searching​

Original video​

Sample and pre-process the cooking video when retrieving​

Cleanup​

What's next?​