#!/usr/bin/env python3 """Run small, asserted query-semantics examples against a disposable database. Uses unique fixture names and removes its own records and indexes afterward. Set ADB_USER and ADB_PASSWORD; see QuerySemantics.rst for the expected outcomes. """ import argparse import json import math import os import struct import uuid from pathlib import Path from aperturedb.Connector import Connector def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--host', required=True) parser.add_argument('--port', type=int, default=55555) parser.add_argument('--no-ssl', action='store_true') parser.add_argument('--output', type=Path) args = parser.parse_args() client = Connector(host=args.host, port=args.port, user=os.environ['ADB_USER'], password=os.environ['ADB_PASSWORD'], use_ssl=not args.no_ssl, retry_max_attempts=1) prefix = 'QueryGuide_' + uuid.uuid4().hex[:12] classes, indexes, sets, checks = [], [], [], [] def request(commands, blobs=None, error=False): response, _ = client.query(commands, blobs or []) success = isinstance(response, list) and all( next(iter(row.values())).get('status') == 0 for row in response) assert success != error, (commands, response) return response def command(name, body, blobs=None): return request([{name: body}], blobs)[0][name] def ingest(suffix, rows): name = prefix + suffix classes.append(name) request([{'AddEntity': {'class': name, 'properties': row}} for row in rows]) return name def index(name, prop, kind): selector = {'target': 'entity', 'class': name, 'property': prop, 'kind': kind} command('CreateIndex', selector) indexes.append(selector) def ids(label, name, expression, expected, **extra): result = command('FindEntity', {'with_class': name, 'constraints': expression, 'results': {'list': ['rid']}, **extra}) actual = sorted(row['rid'] for row in result.get('entities', [])) assert actual == expected, (label, actual, expected) checks.append({'example': label, 'ids': actual}) try: native = ingest('_Native', [{'rid': 0, 'p': 1}]) request([{'AddEntity': {'class': native, 'properties': {'rid': 1, 'p': '1'}}}], error=True) checks.append({'example': 'native class type enforcement', 'expected_error': True}) mixed = ingest('_Json', [{'rid': i, 'p': {'_json': value}} for i, value in enumerate([1, '1', True, None])] + [{'rid': 4}]) for i, value in enumerate([1, '1', True]): ids('guarded JSON scalar ' + str(i), mixed, ['all', ['$rid', '==', i], ['$p', '==', value]], [i]) ids('JSON null and missing', mixed, ['$p', 'is_missing'], [3, 4]) request([{'FindEntity': {'with_class': mixed, 'constraints': ['$p', '==', 1], 'results': {'count': True}}}], error=True) ids('limited scan can stop before a later error', mixed, ['$p', '==', 1], [0], limit=1) index(mixed, 'p', 'type') ids('type index preserves JSON null semantics', mixed, ['$p', 'is_missing'], [3, 4]) arrays = ingest('_Arrays', [{'rid': 0, 'a': [1, '1', True], 'j': {'_json': [9007199254740993]}}, {'rid': 1, 'a': [], 'j': {'_json': [9007199254740992]}}]) for stage in ['scan', 'array index']: if stage == 'array index': index(arrays, 'a', 'array') index(arrays, 'j', 'array') ids('empty left list is a subset: ' + stage, arrays, [{'array': []}, 'in', '$a'], [0, 1]) ids('all elements must be contained: ' + stage, arrays, ['$a', 'in', {'array': [1]}], [1]) ids('exact JSON integer membership: ' + stage, arrays, [{'json': 9007199254740993}, 'in', '$j'], [0]) precision = ingest('_Precision', [{'rid': 0, 'p': {'_json': 9007199254740993}}]) ids('large integer is not a finite scalar number', precision, ['$p', 'is_finite'], []) text = ingest('_Text', [{'rid': i, 'text': value} for i, value in enumerate(['Straße', 'STRASSE', 'ＡＢＣ', 'abc'])]) ids('full case folding', text, ['$text', 'icontains', 'STRASSE'], [0, 1]) ids('regex case matching is different', text, ['$text', 'iregex', '^strasse$'], [1]) index(text, 'text', 'fulltext') score = {'fulltext': ['$text', 'abc']} ids('fulltext compatibility normalization', text, [score, '>', 0], [2, 3], sort={'expression': score, 'order': 'descending'}) ids('similarity does not apply fulltext normalization', text, [{'similarity': ['$text', 'abc']}, '==', 1], [3]) json_text = ingest('_JsonText', [{'rid': 0, 'text': {'_json': 'abc'}}]) index(json_text, 'text', 'fulltext') request([{'FindEntity': {'with_class': json_text, 'constraints': [score, '>', 0], 'results': {'count': True}}}], error=True) checks.append({'example': 'JSON text is not fulltext-native', 'expected_error': True}) name = prefix + '_Vectors' command('AddDescriptorSet', {'name': name, 'dimensions': 2, 'engine': 'Flat', 'metric': 'L2'}) sets.append(name) pack = lambda x: struct.pack('ff', x, 0.0) request([{'AddDescriptor': {'set': name, 'properties': {'rid': i, 'eligible': i >= 2}}} for i in range(4)], [pack(x) for x in [.1, .2, 1, 2]]) command('UpdateDescriptorSet', {'with_name': name, 'build_index': True}) search = {'set': name, 'engine': 'Flat', 'metric': 'L2', 'k_neighbors': 2, 'constraints': ["$eligible", "==", True], 'distances': True, 'results': {'list': ['rid']}} result = command('FindDescriptor', search, [pack(0)]) assert [r['rid'] for r in result['entities']] == [2, 3], result assert all(math.isclose(row['_distance'], expected) for row, expected in zip(result['entities'], [1, 4])), result result = command('FindDescriptor', {**search, 'knn_first': True}, [pack(0)]) assert not result.get('entities'), result # A single transaction prevents the background index builder from # covering the fresh descriptor between insertion and these queries. response = request([ {'AddDescriptor': {'set': name, 'properties': {'rid': 4, 'eligible': True}}}, {'FindDescriptor': search}, {'FindDescriptor': {**search, 'indexed_results_only': True}}], [pack(.5), pack(0), pack(0)]) assert [r['rid'] for r in response[1]['FindDescriptor']['entities']] == [4, 2], response assert [r['rid'] for r in response[2]['FindDescriptor']['entities']] == [2, 3], response checks.append({'example': 'filter-first versus knn-first', 'exact_ids': [2, 3], 'knn_first_ids': [], 'with_fresh_ids': [4, 2], 'indexed_only_ids': [2, 3]}) finally: for selector in reversed(indexes): command('RemoveIndex', selector) for name in sets: command('DeleteDescriptorSet', {'with_name': name}) for name in classes: command('DeleteEntity', {'with_class': name}) report = {'fixture_prefix': prefix, 'examples': checks, 'failures': 0} rendered = json.dumps(report, indent=2, ensure_ascii=False) if args.output: args.output.write_text(rendered + '\n') print(rendered) if __name__ == '__main__': main()