From d162ab3410fae487e95420a82ca7676edbb3ec84 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Tue, 6 Oct 2026 22:00:24 +0800 Subject: [PATCH 1/9] feat(zvec): add DiskANN benchmark adapter - Add DiskANN config, CLI options, and native SDK parameter mappings - Support DiskANN config serialization and preserve HNSW defaults - Validate collection reuse, insert results, and index completeness - Add regression tests and opt-in Linux native smoke test - Document separate index build and search-only workflows Signed-off-by: Wayne Gao --- README.md | 63 +++ tests/test_zvec.py | 367 ++++++++++++++++++ vectordb_bench/backend/clients/__init__.py | 4 +- vectordb_bench/backend/clients/zvec/cli.py | 106 ++++- vectordb_bench/backend/clients/zvec/config.py | 24 +- vectordb_bench/backend/clients/zvec/zvec.py | 131 ++++++- 6 files changed, 670 insertions(+), 25 deletions(-) create mode 100644 tests/test_zvec.py diff --git a/README.md b/README.md index 0ffa98f0e..77989527f 100644 --- a/README.md +++ b/README.md @@ -572,6 +572,8 @@ Options: ### Run Zvec from command line +HNSW remains the default, so existing commands are unchanged: + ```bash vectordbbench zvec --path Performance768D10M --db-label 16c64g-v0.1 \ --case-type Performance768D10M --num-concurrency 12,14,16,18,20 \ @@ -591,6 +593,67 @@ To list the options for zvec, execute vectordbbench zvec --help results ``` + #### Zvec DiskANN + + DiskANN requires a zvec SDK exposing `DiskAnnIndexParam`, `DiskAnnQueryParam`, and `Collection.close()`. + This adapter uses zvec's native C++ DiskANN, not the Microsoft Rust implementation. + From this updated checkout on Linux, in an environment with benchmark dependencies and the DiskANN-enabled SDK + already installed, install the adapter and run its checks: + + ```bash + python -m pip install --no-deps -e . + python -m pip install pytest + vectordbbench zvec --help + python -m pytest tests/test_zvec.py -q + ZVEC_NATIVE_TESTS=1 python -m pytest tests/test_zvec.py -k native_diskann -v + ``` + + The native test builds 1,200 32-dimensional vectors in a temporary directory, checks insertion and index completion, + closes/reopens the collection, searches it, and exercises rebuilding. It is a functionality test, not a recall or + performance benchmark. It is skipped unless explicitly enabled on Linux. Run it successfully before a large benchmark. + + | Option | Default | Meaning | + | --- | --- | --- | + | `--index-type` | hnsw | Select hnsw or diskann | + | `--metric-type` | IP when omitted | IP, COSINE or L2; use the same metric when building and reopening | + | `--max-degree` | 64 | DiskANN graph degree, 1-100 | + | `--build-list-size` | 100 | Construction candidate list, 10-100 | + | `--search-list-size` | 300 | Query candidate list, at least 1; the engine uses at least top-k | + | `--pq-chunk-num` | 0 | PQ bytes per vector; 0 is automatic, otherwise at most min(dimension, 1024) | + + DiskANN stores FP32 vectors and uses internal PQ for graph traversal. HNSW options (`--m`, `--ef-construction`, + `--ef-search`, `--quantize-type`, `--is-using-refiner`) are rejected in DiskANN mode, and DiskANN options are rejected + in HNSW mode. PQ chunks are not equivalent to HNSW INT8 scalar quantization. + + After the checks pass, build a separate DiskANN collection. The load stage includes `optimize()`, which builds the + target index; flushing alone does not establish that a DiskANN index was built. `--drop-old` replaces any existing + DiskANN collection at this path. The adapter refuses to use or delete an existing non-DiskANN collection. + + ```bash + export DATASET_LOCAL_DIR="$PWD/dataset" + vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ + --case-type Performance768D10M --metric-type IP \ + --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ + --drop-old --load --skip-search-serial --skip-search-concurrent + ``` + + Then search the same index without reloading or rebuilding it: + + ```bash + vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ + --case-type Performance768D10M --metric-type IP \ + --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ + --search-list-size 300 --num-concurrency 12,14,16,18,20 \ + --skip-drop-old --skip-load --search-serial --search-concurrent + ``` + + Search-only runs validate the stored dimensions, metric, build parameters, positive document count and complete + indexing. Keep build parameters unchanged when reopening; sweep `--search-list-size` to compare QPS at matched recall. + The examples explicitly use IP, the adapter's historical default. A dataset labelled COSINE does not change that + default or prove that its vectors are normalized. For a COSINE comparison, build both algorithms with + `--metric-type COSINE` in separate paths; do not relabel an existing IP index as COSINE. Multiple search processes can + duplicate in-memory PQ codes and metadata, so measure total process memory as well as QPS. + ### Run Doris from command line Doris supports ann index with type hnsw from version 4.0.x diff --git a/tests/test_zvec.py b/tests/test_zvec.py new file mode 100644 index 000000000..25a9b98f6 --- /dev/null +++ b/tests/test_zvec.py @@ -0,0 +1,367 @@ +import importlib.util +import os +from pathlib import Path +import pickle +import random +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner +from pydantic import ValidationError + +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType, MetricType +from vectordb_bench.backend.clients.zvec.config import ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig +from vectordb_bench.backend.filter import non_filter + + +def test_diskann_config_and_registry_roundtrip(): + config = ZvecDiskANNIndexConfig(metric_type=MetricType.COSINE, pq_chunk_num=96) + assert config.index_param() == {"max_degree": 64, "list_size": 100, "pq_chunk_num": 96} + assert config.search_param() == {"index": IndexType.DISKANN, "list_size": 300} + assert DB.Zvec.case_config_cls(IndexType.DISKANN) is ZvecDiskANNIndexConfig + saved = config.model_dump(mode="json") + assert DB.Zvec.case_config_cls(saved["index"])(**saved) == config + assert pickle.loads(pickle.dumps(config)) == config + assert DB.Zvec.case_config_cls() is ZvecHNSWIndexConfig + assert DB.Zvec.case_config_cls(IndexType.HNSW) is ZvecHNSWIndexConfig + + +@pytest.mark.parametrize( + "values", + [ + {"max_degree": 0}, + {"max_degree": 101}, + {"build_list_size": 9}, + {"build_list_size": 101}, + {"search_list_size": 0}, + {"pq_chunk_num": -1}, + {"pq_chunk_num": 1025}, + {"index": "HNSW"}, + ], +) +def test_diskann_config_rejects_unsupported_values(values): + with pytest.raises(ValidationError): + ZvecDiskANNIndexConfig(**values) + + +@pytest.fixture +def adapter(monkeypatch): + sdk = ModuleType("zvec") + sdk.IndexType = SimpleNamespace(DISKANN="DISKANN", HNSW="HNSW") + sdk.MetricType = SimpleNamespace(IP="IP", COSINE="COSINE", L2="L2") + sdk.QuantizeType = SimpleNamespace(UNDEFINED="UNDEFINED", INT8="INT8", FP16="FP16", INT4="INT4") + sdk.DataType = SimpleNamespace(INT64="INT64", STRING="STRING", VECTOR_FP32="VECTOR_FP32") + sdk.LogLevel = SimpleNamespace(WARN="WARN") + sdk.init = Mock() + sdk.CollectionOption = lambda **kwargs: SimpleNamespace(**kwargs) + sdk.OptimizeOption = lambda: SimpleNamespace() + sdk.InvertIndexParam = lambda **kwargs: SimpleNamespace(**kwargs) + sdk.FieldSchema = lambda *args, **kwargs: SimpleNamespace() + sdk.VectorSchema = lambda name, data_type, **kwargs: SimpleNamespace(name=name, data_type=data_type, **kwargs) + sdk.CollectionSchema = lambda **kwargs: SimpleNamespace(**kwargs) + sdk.Doc = lambda **kwargs: SimpleNamespace(**kwargs) + sdk.VectorQuery = lambda **kwargs: SimpleNamespace(**kwargs) + sdk.DiskAnnIndexParam = lambda **kwargs: SimpleNamespace(type="DISKANN", **kwargs) + sdk.DiskAnnQueryParam = lambda **kwargs: SimpleNamespace(type="DISKANN", **kwargs) + sdk.HnswIndexParam = lambda **kwargs: SimpleNamespace(type="HNSW", **kwargs) + sdk.HnswQueryParam = lambda **kwargs: SimpleNamespace(type="HNSW", **kwargs) + sdk.open = Mock() + sdk.create_and_open = Mock() + monkeypatch.setitem(sys.modules, "zvec", sdk) + source = Path(__file__).parents[1] / "vectordb_bench/backend/clients/zvec/zvec.py" + spec = importlib.util.spec_from_file_location("vectordb_bench.backend.clients.zvec._unit_adapter", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.Zvec, sdk + + +def make_collection(adapter, *, config=None, count=100, completeness=1.0, dimension=768): + client, _sdk = adapter + config = config or ZvecDiskANNIndexConfig(pq_chunk_num=96) + field = SimpleNamespace(dimension=dimension, index_param=client._parse_index_param(config)) + collection = Mock() + collection.schema.vector.return_value = field + collection.stats = SimpleNamespace(doc_count=count, index_completeness={"dense": completeness}) + return collection + + +def make_client(adapter, config=None): + client, _sdk = adapter + instance = object.__new__(client) + instance.case_config = config or ZvecDiskANNIndexConfig(pq_chunk_num=96) + instance.table_name = "test" + instance.dim = 768 + instance.path = "unused" + instance.with_scalar_labels = False + instance._scalar_label_field = "label" + instance.query_param = client._parse_query_param(instance.case_config) + instance.option = SimpleNamespace(read_only=False, enable_mmap=True) + instance.expr = "" + return instance + + +def test_sdk_parameter_mapping_does_not_conflate_build_and_query(adapter): + client, sdk = adapter + config = ZvecDiskANNIndexConfig(metric_type=MetricType.COSINE, pq_chunk_num=96) + index = client._parse_index_param(config) + query = client._parse_query_param(config) + assert index.type == sdk.IndexType.DISKANN + assert index.metric_type == sdk.MetricType.COSINE + assert index.quantize_type == sdk.QuantizeType.UNDEFINED + assert (index.max_degree, index.list_size, index.pq_chunk_num) == (64, 100, 96) + assert query.list_size == 300 + hnsw = ZvecHNSWIndexConfig(M=50, ef_construction=500, ef_search=118, quantize_type="int8", is_using_refiner=True) + index = client._parse_index_param(hnsw) + query = client._parse_query_param(hnsw) + assert (index.type, index.m, index.ef_construction, index.quantize_type) == ("HNSW", 50, 500, "INT8") + assert query.ef == 118 and query.is_using_refiner + assert index.metric_type == sdk.MetricType.IP + + +@pytest.mark.parametrize( + "symbol, method", + [ + ("DiskAnnIndexParam", "_parse_index_param"), + ("DiskAnnQueryParam", "_parse_query_param"), + ], +) +def test_diskann_reports_missing_sdk_support(adapter, monkeypatch, symbol, method): + client, sdk = adapter + monkeypatch.delattr(sdk, symbol) + with pytest.raises(ValueError, match=symbol): + getattr(client, method)(ZvecDiskANNIndexConfig()) + assert client._parse_query_param(ZvecHNSWIndexConfig()).type == "HNSW" + + +def test_diskann_creation_rejects_hnsw_directory_before_delete(adapter, tmp_path): + client, sdk = adapter + collection = make_collection(adapter) + collection.schema.vector.return_value.index_param.type = "HNSW" + sdk.open.return_value = collection + with pytest.raises(ValueError, match="non-DiskANN"): + client(768, {"path": str(tmp_path)}, ZvecDiskANNIndexConfig(), drop_old=True) + collection.destroy.assert_not_called() + collection.close.assert_called_once() + sdk.create_and_open.assert_not_called() + + +def test_diskann_rebuild_allows_changed_build_parameters(adapter, tmp_path): + client, sdk = adapter + collection = make_collection(adapter) + sdk.open.return_value = collection + config = ZvecDiskANNIndexConfig(max_degree=32, pq_chunk_num=48) + client(768, {"path": str(tmp_path)}, config, drop_old=True) + assert not sdk.open.call_args.kwargs["option"].read_only + collection.destroy.assert_called_once() + collection.close.assert_called_once() + sdk.create_and_open.assert_called_once() + sdk.create_and_open.return_value.close.assert_called_once() + + +def test_diskann_validates_pq_dimensions_before_open(adapter, tmp_path): + client, sdk = adapter + with pytest.raises(ValueError, match="pq_chunk_num"): + client(8, {"path": str(tmp_path)}, ZvecDiskANNIndexConfig(pq_chunk_num=96), drop_old=True) + sdk.open.assert_not_called() + sdk.create_and_open.assert_not_called() + + +def test_diskann_new_directory_and_search_only_reopen(adapter, tmp_path): + client, sdk = adapter + path = tmp_path / "diskann" + config = ZvecDiskANNIndexConfig(pq_chunk_num=96) + with pytest.raises(ValueError, match="does not exist"): + client(768, {"path": str(path)}, config, drop_old=False) + client(768, {"path": str(path)}, config, drop_old=True) + sdk.create_and_open.assert_called_once() + sdk.create_and_open.return_value.close.assert_called_once() + path.mkdir() + collection = make_collection(adapter, config=config) + sdk.open.return_value = collection + client(768, {"path": str(path)}, config, drop_old=False) + assert sdk.open.call_args.kwargs["option"].read_only + collection.destroy.assert_not_called() + collection.close.assert_called_once() + + +@pytest.mark.parametrize("change", ["metric", "pq", "degree", "dimension", "incomplete", "empty"]) +def test_diskann_search_only_checks_persisted_schema_and_readiness(adapter, tmp_path, change): + client, sdk = adapter + collection = make_collection(adapter) + field = collection.schema.vector.return_value + if change == "metric": + field.index_param.metric_type = "COSINE" + elif change == "pq": + field.index_param.pq_chunk_num = 48 + elif change == "degree": + field.index_param.max_degree = 32 + elif change == "dimension": + field.dimension = 128 + elif change == "incomplete": + collection.stats.index_completeness["dense"] = 0.5 + else: + collection.stats.doc_count = 0 + sdk.open.return_value = collection + with pytest.raises((ValueError, RuntimeError)): + client(768, {"path": str(tmp_path)}, ZvecDiskANNIndexConfig(pq_chunk_num=96), drop_old=False) + collection.close.assert_called_once() + collection.destroy.assert_not_called() + + +def test_diskann_session_closes_on_error(adapter): + instance = make_client(adapter) + collection = make_collection(adapter) + adapter[1].open.return_value = collection + with pytest.raises(RuntimeError, match="test body"): + with instance.init(): + raise RuntimeError("test body") + collection.close.assert_called_once() + assert instance.collection is None + + +def test_diskann_insert_checks_partial_and_malformed_statuses(adapter): + instance = make_client(adapter) + instance.collection = Mock() + good = SimpleNamespace(ok=lambda: True) + bad = SimpleNamespace(ok=lambda: False) + instance.collection.insert.return_value = [good, bad] + count, error = instance.insert_embeddings([[0.0], [1.0]], [10, 11]) + assert count == 1 and error.non_retryable and error.inserted_count == 1 + instance.collection.insert.return_value = [good] + count, error = instance.insert_embeddings([[0.0], [1.0]], [10, 11]) + assert count == 0 and error.non_retryable + instance.collection.insert.return_value = [good, good] + assert instance.insert_embeddings([[0.0], [1.0]], [10, 11]) == (2, None) + count, error = instance.insert_embeddings([[0.0]], [10, 11]) + assert count == 0 and error.non_retryable + + +def test_diskann_optimize_requires_exact_count_and_index_completion(adapter): + instance = make_client(adapter) + instance.collection = make_collection(adapter, count=99, completeness=0.0) + with pytest.raises(RuntimeError, match="expected 100"): + instance.optimize(data_size=100) + instance.collection.optimize.assert_not_called() + instance.collection.stats.doc_count = 100 + with pytest.raises(RuntimeError, match="not fully indexed"): + instance.optimize(data_size=100) + instance.collection.optimize.side_effect = lambda **_kwargs: instance.collection.stats.index_completeness.update( + dense=1.0 + ) + instance.optimize(data_size=100) + assert instance.collection.stats.index_completeness["dense"] == 1.0 + + +def test_search_passes_diskann_query_parameters_without_refiner(adapter): + instance = make_client(adapter) + instance.collection = Mock() + instance.collection.query.return_value = [SimpleNamespace(id="7"), SimpleNamespace(id="8")] + assert instance.search_embedding([0.1] * 768, k=2) == [7, 8] + arguments = instance.collection.query.call_args.kwargs + assert arguments["vectors"].param.type == "DISKANN" + assert arguments["vectors"].param.list_size == 300 + assert arguments["topk"] == 2 and arguments["output_fields"] == [] + + +def test_cli_defaults_and_diskann_selection(monkeypatch): + from vectordb_bench.backend.clients.zvec import cli as zvec_cli + + captured = {} + monkeypatch.setattr(zvec_cli, "run", lambda **values: captured.update(values)) + runner = CliRunner() + result = runner.invoke(zvec_cli.Zvec, ["--path", "unused", "--dry-run"]) + assert result.exit_code == 0, result.output + assert isinstance(captured["db_case_config"], ZvecHNSWIndexConfig) + assert captured["db_case_config"] == ZvecHNSWIndexConfig() + result = runner.invoke( + zvec_cli.Zvec, + [ + "--path", + "unused", + "--index-type", + "diskann", + "--metric-type", + "cosine", + "--max-degree", + "64", + "--build-list-size", + "100", + "--pq-chunk-num", + "96", + "--search-list-size", + "400", + "--case-type", + "Performance768D10M", + "--skip-drop-old", + "--skip-load", + "--dry-run", + ], + ) + assert result.exit_code == 0, result.output + config = captured["db_case_config"] + assert isinstance(config, ZvecDiskANNIndexConfig) + assert config.metric_type == MetricType.COSINE + assert config.search_list_size == 400 and config.build_list_size == 100 + assert captured["drop_old"] is False and captured["load"] is False + + +@pytest.mark.parametrize( + "arguments", + [ + ["--index-type", "diskann", "--ef-search", "118"], + ["--index-type", "diskann", "--m", "50"], + ["--index-type", "diskann", "--quantize-type", "int8"], + ["--index-type", "diskann", "--is-using-refiner"], + ["--index-type", "hnsw", "--pq-chunk-num", "96"], + ["--index-type", "diskann", "--max-degree", "101"], + ["--index-type", "diskann", "--search-list-size", "0"], + ], +) +def test_cli_rejects_mixed_or_invalid_parameters(monkeypatch, arguments): + from vectordb_bench.backend.clients.zvec import cli as zvec_cli + + run = Mock() + monkeypatch.setattr(zvec_cli, "run", run) + result = CliRunner().invoke(zvec_cli.Zvec, ["--path", "unused", "--dry-run", *arguments]) + assert result.exit_code == 2, result.output + run.assert_not_called() + + +@pytest.mark.skipif( + sys.platform != "linux" or os.getenv("ZVEC_NATIVE_TESTS") != "1", + reason="Opt-in Linux DiskANN smoke test: ZVEC_NATIVE_TESTS=1", +) +def test_native_diskann_build_reopen_and_search(tmp_path): + import zvec as sdk + + from vectordb_bench.backend.clients.zvec.zvec import Zvec + + if not hasattr(sdk, "DiskAnnIndexParam"): + pytest.fail("Installed zvec must support DiskAnnIndexParam") + if not callable(getattr(sdk.Collection, "close", None)): + pytest.fail("Installed zvec must support Collection.close()") + random_values = random.Random(71) + vectors = [[random_values.random() for _ in range(32)] for _ in range(1200)] + config = ZvecDiskANNIndexConfig(metric_type=MetricType.L2, max_degree=32, pq_chunk_num=8) + connection = {"path": str(tmp_path / "diskann")} + client = Zvec(32, connection, config, drop_old=True) + with client.init(): + for start in range(0, len(vectors), 100): + batch = vectors[start : start + 100] + count, error = client.insert_embeddings(batch, list(range(start, start + len(batch)))) + assert error is None and count == len(batch) + client.optimize(data_size=len(vectors)) + assert client.collection.schema.vector("dense").index_param.type == sdk.IndexType.DISKANN + reopened = Zvec(32, connection, config, drop_old=False) + reopened.prepare_filter(non_filter) + with reopened.init(): + assert reopened.collection.stats.doc_count == len(vectors) + result = reopened.search_embedding(vectors[17], k=10) + assert len(result) == 10 and len(set(result)) == 10 + assert 17 in result + rebuilt = Zvec(32, connection, config, drop_old=True) + with rebuilt.init(): + assert rebuilt.collection.stats.doc_count == 0 diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index dbd83bb66..127e523c9 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -663,8 +663,10 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 return HologresIndexConfig if self == DB.Zvec: - from .zvec.config import ZvecHNSWIndexConfig + from .zvec.config import ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig + if index_type == IndexType.DISKANN: + return ZvecDiskANNIndexConfig return ZvecHNSWIndexConfig if self == DB.TencentElasticsearch: diff --git a/vectordb_bench/backend/clients/zvec/cli.py b/vectordb_bench/backend/clients/zvec/cli.py index 0a9e09761..b58efb27a 100644 --- a/vectordb_bench/backend/clients/zvec/cli.py +++ b/vectordb_bench/backend/clients/zvec/cli.py @@ -1,6 +1,7 @@ from typing import Annotated, Unpack import click +from click.core import ParameterSource from ....cli.cli import ( CommonTypedDict, @@ -8,7 +9,8 @@ click_parameter_decorators_from_typed_dict, run, ) -from .. import DB +from .. import DB, MetricType +from .config import ZvecConfig, ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig class ZvecTypedDict(CommonTypedDict): @@ -23,7 +25,7 @@ class ZvecHNSWTypedDict(CommonTypedDict, ZvecTypedDict): int, click.option("--m", type=int, default=50, help="HNSW index parameter m."), ] - ef_construct: Annotated[ + ef_construction: Annotated[ int, click.option("--ef-construction", type=int, default=500, help="HNSW index parameter ef_construction"), ] @@ -32,7 +34,7 @@ class ZvecHNSWTypedDict(CommonTypedDict, ZvecTypedDict): click.option("--ef-search", type=int, default=300, help="HNSW index parameter ef for search"), ] quantize_type: Annotated[ - int, + str, click.option("--quantize-type", type=str, default="", help="HNSW index quantize type, fp16/int8 supported"), ] is_using_refiner: Annotated[ @@ -47,24 +49,100 @@ class ZvecHNSWTypedDict(CommonTypedDict, ZvecTypedDict): ] -# default to hnsw +class ZvecAlgorithmTypedDict(ZvecHNSWTypedDict): + index_type: Annotated[ + str, + click.option( + "--index-type", + type=click.Choice(["hnsw", "diskann"], case_sensitive=False), + default="hnsw", + show_default=True, + help="Zvec vector index algorithm.", + ), + ] + metric_type: Annotated[ + str | None, + click.option( + "--metric-type", + type=click.Choice(["IP", "COSINE", "L2"], case_sensitive=False), + default=None, + help="Distance metric; omitted keeps the existing IP default.", + ), + ] + max_degree: Annotated[ + int, + click.option("--max-degree", type=click.IntRange(1, 100), default=64, help="DiskANN graph out-degree."), + ] + build_list_size: Annotated[ + int, + click.option( + "--build-list-size", + type=click.IntRange(10, 100), + default=100, + help="DiskANN candidate list size during construction.", + ), + ] + search_list_size: Annotated[ + int, + click.option( + "--search-list-size", + type=click.IntRange(min=1), + default=300, + help="DiskANN query candidate list size; the engine uses at least top-k.", + ), + ] + pq_chunk_num: Annotated[ + int, + click.option( + "--pq-chunk-num", + type=click.IntRange(0, 1024), + default=0, + help="DiskANN PQ bytes per vector; 0 selects automatically, nonzero must not exceed dimension.", + ), + ] + + @cli.command() -@click_parameter_decorators_from_typed_dict(ZvecHNSWTypedDict) -def Zvec(**parameters: Unpack[ZvecHNSWTypedDict]): - from .config import ZvecConfig, ZvecHNSWIndexConfig +@click_parameter_decorators_from_typed_dict(ZvecAlgorithmTypedDict) +def Zvec(**parameters: Unpack[ZvecAlgorithmTypedDict]): + index_type = parameters["index_type"].lower() + context = click.get_current_context() + unsupported = ( + ("m", "ef_construction", "ef_search", "quantize_type", "is_using_refiner") + if index_type == "diskann" + else ("max_degree", "build_list_size", "search_list_size", "pq_chunk_num") + ) + for parameter in unsupported: + if context.get_parameter_source(parameter) not in (None, ParameterSource.DEFAULT): + message = f"--{parameter.replace('_', '-')} does not apply to {index_type}" + raise click.UsageError(message) - run( - db=DB.Zvec, - db_config=ZvecConfig( - db_label=parameters["db_label"], - path=parameters["path"], - ), - db_case_config=ZvecHNSWIndexConfig( + metric_type = parameters["metric_type"] + metric_type = MetricType(metric_type.upper()) if metric_type else None + if index_type == "diskann": + case_config = ZvecDiskANNIndexConfig( + metric_type=metric_type, + max_degree=parameters["max_degree"], + build_list_size=parameters["build_list_size"], + search_list_size=parameters["search_list_size"], + pq_chunk_num=parameters["pq_chunk_num"], + ) + else: + case_config = ZvecHNSWIndexConfig( + metric_type=metric_type, M=parameters["m"], ef_construction=parameters["ef_construction"], ef_search=parameters["ef_search"], quantize_type=parameters["quantize_type"], is_using_refiner=parameters["is_using_refiner"], + ) + + run( + db=DB.Zvec, + db_config=ZvecConfig( + db_label=parameters["db_label"], + path=parameters["path"], ), + db_case_config=case_config, **parameters, ) diff --git a/vectordb_bench/backend/clients/zvec/config.py b/vectordb_bench/backend/clients/zvec/config.py index 4c649d08a..0d4f453df 100644 --- a/vectordb_bench/backend/clients/zvec/config.py +++ b/vectordb_bench/backend/clients/zvec/config.py @@ -1,6 +1,8 @@ -from pydantic import BaseModel +from typing import Literal -from ..api import DBCaseConfig, DBConfig, MetricType +from pydantic import BaseModel, Field + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType class ZvecConfig(DBConfig): @@ -34,3 +36,21 @@ class ZvecHNSWIndexConfig(ZvecIndexConfig): quantize_type: str = "" is_using_refiner: bool = False + + +class ZvecDiskANNIndexConfig(ZvecIndexConfig): + index: Literal[IndexType.DISKANN] = IndexType.DISKANN + max_degree: int = Field(default=64, ge=1, le=100) + build_list_size: int = Field(default=100, ge=10, le=100) + search_list_size: int = Field(default=300, ge=1) + pq_chunk_num: int = Field(default=0, ge=0, le=1024) + + def index_param(self) -> dict: + return { + "max_degree": self.max_degree, + "list_size": self.build_list_size, + "pq_chunk_num": self.pq_chunk_num, + } + + def search_param(self) -> dict: + return {"index": self.index, "list_size": self.search_list_size} diff --git a/vectordb_bench/backend/clients/zvec/zvec.py b/vectordb_bench/backend/clients/zvec/zvec.py index 63cd87c9b..037a05511 100644 --- a/vectordb_bench/backend/clients/zvec/zvec.py +++ b/vectordb_bench/backend/clients/zvec/zvec.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import logging from contextlib import contextmanager +from pathlib import Path import zvec from zvec import ( @@ -18,8 +21,8 @@ from vectordb_bench.backend.filter import Filter, FilterOp -from ..api import MetricType, VectorDB -from .config import ZvecConfig, ZvecHNSWIndexConfig, ZvecIndexConfig +from ..api import MetricType, NonRetryableInsertError, PartialInsertError, VectorDB +from .config import ZvecConfig, ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig, ZvecIndexConfig log = logging.getLogger(__name__) @@ -49,6 +52,7 @@ def __init__( self.table_name = collection_name self.dim = dim self.path = db_config["path"] + self.expr = "" # avoid the search_param being called every time during the search process self.search_config = db_case_config.search_param() self._scalar_id_field = "id" @@ -88,6 +92,13 @@ def __init__( self.query_param = Zvec._parse_query_param(db_case_config) + if isinstance(db_case_config, ZvecDiskANNIndexConfig): + if dim <= 0 or db_case_config.pq_chunk_num > dim: + message = "DiskANN requires a positive dimension and pq_chunk_num <= dimension" + raise ValueError(message) + self._prepare_diskann_collection(drop_old) + return + if drop_old: try: collection = zvec.open(self.path) @@ -99,11 +110,70 @@ def __init__( else: collection = zvec.open(self.path) + def _prepare_diskann_collection(self, drop_old: bool) -> None: + if Path(self.path).exists(): + option = CollectionOption(read_only=not drop_old, enable_mmap=True) + collection = zvec.open(self.path, option=option) + try: + self._validate_diskann_collection(collection, require_ready=not drop_old, rebuilding=drop_old) + if drop_old: + collection.destroy() + finally: + collection.close() + elif not drop_old: + message = f"DiskANN collection does not exist at {self.path}; build it before search-only benchmarking" + raise ValueError(message) + if drop_old: + collection = zvec.create_and_open(path=self.path, schema=self.schema, option=self.option) + collection.close() + log.info( + "DiskANN selected: path=%s metric=%s build=%s search=%s stored_vectors=FP32", + self.path, + self._parse_metric(self.case_config.metric_type), + self.case_config.index_param(), + self.search_config, + ) + + def _validate_diskann_collection( + self, + collection: zvec.Collection, + *, + require_ready: bool, + rebuilding: bool = False, + ) -> None: + field = collection.schema.vector("dense") + if field is None or field.index_param.type != zvec.IndexType.DISKANN: + message = "Refusing to use or delete a non-DiskANN collection; choose a separate DiskANN path" + raise ValueError(message) + if rebuilding: + return + expected = self._parse_index_param(self.case_config) + actual = field.index_param + if field.dimension != self.dim or any( + getattr(actual, name) != getattr(expected, name) + for name in ("metric_type", "quantize_type", "max_degree", "list_size", "pq_chunk_num") + ): + message = "Stored DiskANN schema differs from requested dimensions, metric or build parameters" + raise ValueError(message) + if require_ready: + stats = collection.stats + if stats.doc_count <= 0 or stats.index_completeness.get("dense") != 1.0: + message = "DiskANN collection is empty or not fully indexed; complete load and optimize first" + raise RuntimeError(message) + @contextmanager def init(self): self.collection = zvec.open(self.path, self.option) - yield - self.collection = None + try: + if isinstance(self.case_config, ZvecDiskANNIndexConfig): + self._validate_diskann_collection(self.collection, require_ready=self.option.read_only) + yield + finally: + try: + if isinstance(self.case_config, ZvecDiskANNIndexConfig): + self.collection.close() + finally: + self.collection = None def insert_embeddings( self, @@ -111,7 +181,12 @@ def insert_embeddings( metadata: list[int], labels_data: list[str] | None = None, **kwargs, - ) -> tuple[int, Exception]: + ) -> tuple[int, Exception | None]: + if isinstance(self.case_config, ZvecDiskANNIndexConfig) and ( + len(embeddings) != len(metadata) + or (self.with_scalar_labels and (labels_data is None or len(labels_data) != len(metadata))) + ): + return 0, NonRetryableInsertError("Embedding, ID and label batch lengths must match") docs = [] for i, id_ in enumerate(metadata): embedding = embeddings[i] @@ -128,7 +203,14 @@ def insert_embeddings( ) ) try: - self.collection.insert(docs) + statuses = self.collection.insert(docs) + if isinstance(self.case_config, ZvecDiskANNIndexConfig): + if len(statuses) != len(docs): + return 0, NonRetryableInsertError("zvec insert returned an incomplete status list") + inserted = sum(status.ok() for status in statuses) + if inserted != len(docs): + message = f"zvec DiskANN batch insert failed for {len(docs) - inserted}/{len(docs)} documents" + return inserted, PartialInsertError(message, inserted_count=inserted) return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into Zvec table ({self.table_name}), error: {e}") @@ -153,7 +235,24 @@ def search_embedding( return [int(result.id) for result in results] def optimize(self, data_size: int | None = None): + if isinstance(self.case_config, ZvecDiskANNIndexConfig): + self._validate_diskann_collection(self.collection, require_ready=False) + if data_size is not None and self.collection.stats.doc_count != data_size: + message = f"DiskANN document count {self.collection.stats.doc_count} differs from expected {data_size}" + raise RuntimeError(message) + log.info("Building DiskANN index via optimize: documents=%s", self.collection.stats.doc_count) self.collection.optimize(option=OptimizeOption()) + if isinstance(self.case_config, ZvecDiskANNIndexConfig): + self._validate_diskann_collection(self.collection, require_ready=True) + stats = self.collection.stats + if data_size is not None and stats.doc_count != data_size: + message = f"DiskANN document count after optimize is {stats.doc_count}, expected {data_size}" + raise RuntimeError(message) + log.info( + "DiskANN optimize completed: documents=%s index_completeness=%s", + stats.doc_count, + stats.index_completeness["dense"], + ) def prepare_filter(self, filters: Filter): self.option = CollectionOption(read_only=True, enable_mmap=True) @@ -181,7 +280,18 @@ def _parse_metric(cls, metric_type: MetricType) -> zvec.MetricType: return d[metric_type] @classmethod - def _parse_index_param(cls, index_config: ZvecIndexConfig) -> zvec.HnswIndexParam: + def _parse_index_param(cls, index_config: ZvecIndexConfig) -> zvec.HnswIndexParam | zvec.DiskAnnIndexParam: + if isinstance(index_config, ZvecDiskANNIndexConfig): + if not hasattr(zvec, "DiskAnnIndexParam"): + message = "Installed zvec SDK does not expose DiskAnnIndexParam; install a DiskANN-enabled zvec build" + raise ValueError(message) + return zvec.DiskAnnIndexParam( + metric_type=Zvec._parse_metric(index_config.metric_type), + max_degree=index_config.max_degree, + list_size=index_config.build_list_size, + pq_chunk_num=index_config.pq_chunk_num, + quantize_type=QuantizeType.UNDEFINED, + ) if isinstance(index_config, ZvecHNSWIndexConfig): return zvec.HnswIndexParam( metric_type=Zvec._parse_metric(index_config.metric_type), @@ -193,7 +303,12 @@ def _parse_index_param(cls, index_config: ZvecIndexConfig) -> zvec.HnswIndexPara raise ValueError(message) @classmethod - def _parse_query_param(cls, index_config: ZvecIndexConfig) -> zvec.HnswQueryParam: + def _parse_query_param(cls, index_config: ZvecIndexConfig) -> zvec.HnswQueryParam | zvec.DiskAnnQueryParam: + if isinstance(index_config, ZvecDiskANNIndexConfig): + if not hasattr(zvec, "DiskAnnQueryParam"): + message = "Installed zvec SDK does not expose DiskAnnQueryParam; install a DiskANN-enabled zvec build" + raise ValueError(message) + return zvec.DiskAnnQueryParam(list_size=index_config.search_list_size) if isinstance(index_config, ZvecHNSWIndexConfig): return zvec.HnswQueryParam( ef=index_config.ef_search, From 3e41d834c2e0cc7f6897138b37e4824cfc689f4e Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Tue, 6 Oct 2026 22:19:27 +0800 Subject: [PATCH 2/9] fix(zvec): avoid closing destroyed DiskANN collections - Skip close() after successful destruction during index rebuild - Preserve cleanup and abort recreation when destruction fails - Add regression tests for successful and failed destruction Signed-off-by: Wayne Gao --- tests/test_zvec.py | 19 ++++++++++++++++++- vectordb_bench/backend/clients/zvec/zvec.py | 5 ++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/test_zvec.py b/tests/test_zvec.py index 25a9b98f6..18f132f2a 100644 --- a/tests/test_zvec.py +++ b/tests/test_zvec.py @@ -151,16 +151,33 @@ def test_diskann_creation_rejects_hnsw_directory_before_delete(adapter, tmp_path def test_diskann_rebuild_allows_changed_build_parameters(adapter, tmp_path): client, sdk = adapter collection = make_collection(adapter) + + def mark_destroyed(): + collection.close.side_effect = ValueError("collection is already destroyed.") + + collection.destroy.side_effect = mark_destroyed sdk.open.return_value = collection config = ZvecDiskANNIndexConfig(max_degree=32, pq_chunk_num=48) client(768, {"path": str(tmp_path)}, config, drop_old=True) assert not sdk.open.call_args.kwargs["option"].read_only collection.destroy.assert_called_once() - collection.close.assert_called_once() + collection.close.assert_not_called() sdk.create_and_open.assert_called_once() sdk.create_and_open.return_value.close.assert_called_once() +def test_diskann_failed_destroy_closes_collection_without_recreating(adapter, tmp_path): + client, sdk = adapter + collection = make_collection(adapter) + collection.destroy.side_effect = ValueError("destroy failed") + sdk.open.return_value = collection + with pytest.raises(ValueError, match="destroy failed"): + client(768, {"path": str(tmp_path)}, ZvecDiskANNIndexConfig(), drop_old=True) + collection.destroy.assert_called_once() + collection.close.assert_called_once() + sdk.create_and_open.assert_not_called() + + def test_diskann_validates_pq_dimensions_before_open(adapter, tmp_path): client, sdk = adapter with pytest.raises(ValueError, match="pq_chunk_num"): diff --git a/vectordb_bench/backend/clients/zvec/zvec.py b/vectordb_bench/backend/clients/zvec/zvec.py index 037a05511..a96daaab4 100644 --- a/vectordb_bench/backend/clients/zvec/zvec.py +++ b/vectordb_bench/backend/clients/zvec/zvec.py @@ -114,12 +114,15 @@ def _prepare_diskann_collection(self, drop_old: bool) -> None: if Path(self.path).exists(): option = CollectionOption(read_only=not drop_old, enable_mmap=True) collection = zvec.open(self.path, option=option) + destroyed = False try: self._validate_diskann_collection(collection, require_ready=not drop_old, rebuilding=drop_old) if drop_old: collection.destroy() + destroyed = True finally: - collection.close() + if not destroyed: + collection.close() elif not drop_old: message = f"DiskANN collection does not exist at {self.path}; build it before search-only benchmarking" raise ValueError(message) From 54240916a1286551a2b19a0c66e3293b8abc9203 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 24 Oct 2026 15:00:20 +0800 Subject: [PATCH 3/9] fix(zvec): remove ineffective CLI metric override Remove --metric-type because task assembly determines the effective metric from the selected dataset. - Clarify dataset-derived metrics in CLI help and usage examples - Add HNSW/DiskANN regression coverage for COSINE and L2 task assembly - Verify the removed option is rejected rather than silently ignored - Preserve existing HNSW defaults and shared assembler behavior Signed-off-by: Wayne Gao --- README.md | 18 +++--- tests/test_zvec.py | 75 +++++++++++++++++++++- vectordb_bench/backend/clients/zvec/cli.py | 16 +---- 3 files changed, 84 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 77989527f..36d32e78b 100644 --- a/README.md +++ b/README.md @@ -572,7 +572,10 @@ Options: ### Run Zvec from command line -HNSW remains the default, so existing commands are unchanged: +HNSW remains the default. For both HNSW and DiskANN, the distance metric is determined by the selected dataset during task +assembly, not by a Zvec CLI option. For example, `Performance768D10M` uses COSINE. + +Existing HNSW commands are unchanged: ```bash vectordbbench zvec --path Performance768D10M --db-label 16c64g-v0.1 \ @@ -615,7 +618,6 @@ To list the options for zvec, execute vectordbbench zvec --help | Option | Default | Meaning | | --- | --- | --- | | `--index-type` | hnsw | Select hnsw or diskann | - | `--metric-type` | IP when omitted | IP, COSINE or L2; use the same metric when building and reopening | | `--max-degree` | 64 | DiskANN graph degree, 1-100 | | `--build-list-size` | 100 | Construction candidate list, 10-100 | | `--search-list-size` | 300 | Query candidate list, at least 1; the engine uses at least top-k | @@ -632,7 +634,7 @@ To list the options for zvec, execute vectordbbench zvec --help ```bash export DATASET_LOCAL_DIR="$PWD/dataset" vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ - --case-type Performance768D10M --metric-type IP \ + --case-type Performance768D10M \ --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ --drop-old --load --skip-search-serial --skip-search-concurrent ``` @@ -641,7 +643,7 @@ To list the options for zvec, execute vectordbbench zvec --help ```bash vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ - --case-type Performance768D10M --metric-type IP \ + --case-type Performance768D10M \ --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ --search-list-size 300 --num-concurrency 12,14,16,18,20 \ --skip-drop-old --skip-load --search-serial --search-concurrent @@ -649,10 +651,10 @@ To list the options for zvec, execute vectordbbench zvec --help Search-only runs validate the stored dimensions, metric, build parameters, positive document count and complete indexing. Keep build parameters unchanged when reopening; sweep `--search-list-size` to compare QPS at matched recall. - The examples explicitly use IP, the adapter's historical default. A dataset labelled COSINE does not change that - default or prove that its vectors are normalized. For a COSINE comparison, build both algorithms with - `--metric-type COSINE` in separate paths; do not relabel an existing IP index as COSINE. Multiple search processes can - duplicate in-memory PQ codes and metadata, so measure total process memory as well as QPS. + These examples use the Cohere dataset's COSINE metric for both building and searching. For an HNSW/DiskANN + comparison, use the same dataset and separate collection paths; do not relabel an existing IP index as COSINE. + Multiple search processes can duplicate in-memory PQ codes and metadata, so measure total process memory as well + as QPS. ### Run Doris from command line diff --git a/tests/test_zvec.py b/tests/test_zvec.py index 18f132f2a..1053663ce 100644 --- a/tests/test_zvec.py +++ b/tests/test_zvec.py @@ -300,8 +300,6 @@ def test_cli_defaults_and_diskann_selection(monkeypatch): "unused", "--index-type", "diskann", - "--metric-type", - "cosine", "--max-degree", "64", "--build-list-size", @@ -320,11 +318,82 @@ def test_cli_defaults_and_diskann_selection(monkeypatch): assert result.exit_code == 0, result.output config = captured["db_case_config"] assert isinstance(config, ZvecDiskANNIndexConfig) - assert config.metric_type == MetricType.COSINE + assert config.metric_type is None assert config.search_list_size == 400 and config.build_list_size == 100 assert captured["drop_old"] is False and captured["load"] is False +def test_cli_help_documents_dataset_metric(): + from vectordb_bench.backend.clients.zvec import cli as zvec_cli + + result = CliRunner().invoke(zvec_cli.Zvec, ["--help"]) + assert result.exit_code == 0, result.output + assert "--metric-type" not in result.output + assert "selected dataset's distance metric" in " ".join(result.output.split()) + + +@pytest.mark.parametrize("index_type", ["hnsw", "diskann"]) +@pytest.mark.parametrize("metric_type", ["IP", "COSINE", "L2"]) +def test_cli_rejects_metric_override(monkeypatch: pytest.MonkeyPatch, index_type: str, metric_type: str): + from vectordb_bench.backend.clients.zvec import cli as zvec_cli + + run = Mock() + monkeypatch.setattr(zvec_cli, "run", run) + result = CliRunner().invoke( + zvec_cli.Zvec, + ["--path", "unused", "--index-type", index_type, "--metric-type", metric_type, "--dry-run"], + ) + assert result.exit_code == 2, result.output + assert "No such option: --metric-type" in result.output + run.assert_not_called() + + +@pytest.mark.parametrize( + ("index_type", "config_cls"), + [("hnsw", ZvecHNSWIndexConfig), ("diskann", ZvecDiskANNIndexConfig)], +) +@pytest.mark.parametrize( + ("case_type", "expected_metric"), + [("Performance768D10M", MetricType.COSINE), ("Performance768D100M", MetricType.L2)], +) +def test_cli_task_uses_dataset_metric_after_assembly( + monkeypatch: pytest.MonkeyPatch, + index_type: str, + config_cls: type[ZvecHNSWIndexConfig] | type[ZvecDiskANNIndexConfig], + case_type: str, + expected_metric: MetricType, +): + from vectordb_bench.backend.assembler import Assembler + from vectordb_bench.backend.clients.zvec import cli as zvec_cli + from vectordb_bench.backend.data_source import DatasetSource + from vectordb_bench.cli import cli as common_cli + from vectordb_bench.models import TaskConfig + + captured_tasks: list[TaskConfig] = [] + + def capture_tasks(tasks: list[TaskConfig], _task_label: str | None) -> bool: + captured_tasks.extend(tasks) + return True + + monkeypatch.setattr(common_cli.benchmark_runner, "run", capture_tasks) + monkeypatch.setattr(common_cli.benchmark_runner, "has_running", lambda: False) + result = CliRunner().invoke( + zvec_cli.Zvec, + ["--path", "unused", "--index-type", index_type, "--case-type", case_type, "--skip-drop-old", "--skip-load"], + ) + assert result.exit_code == 0, result.output + assert len(captured_tasks) == 1 + task = captured_tasks[0] + assert isinstance(task.db_case_config, config_cls) + assert task.db_case_config.metric_type is None + + runner = Assembler.assemble("test-zvec-metric", task, DatasetSource.S3) + + assert runner.ca.dataset.data.metric_type == expected_metric + assert runner.config.db_case_config.metric_type == expected_metric + assert runner.db is None + + @pytest.mark.parametrize( "arguments", [ diff --git a/vectordb_bench/backend/clients/zvec/cli.py b/vectordb_bench/backend/clients/zvec/cli.py index b58efb27a..00506690d 100644 --- a/vectordb_bench/backend/clients/zvec/cli.py +++ b/vectordb_bench/backend/clients/zvec/cli.py @@ -9,7 +9,7 @@ click_parameter_decorators_from_typed_dict, run, ) -from .. import DB, MetricType +from .. import DB from .config import ZvecConfig, ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig @@ -60,15 +60,6 @@ class ZvecAlgorithmTypedDict(ZvecHNSWTypedDict): help="Zvec vector index algorithm.", ), ] - metric_type: Annotated[ - str | None, - click.option( - "--metric-type", - type=click.Choice(["IP", "COSINE", "L2"], case_sensitive=False), - default=None, - help="Distance metric; omitted keeps the existing IP default.", - ), - ] max_degree: Annotated[ int, click.option("--max-degree", type=click.IntRange(1, 100), default=64, help="DiskANN graph out-degree."), @@ -105,6 +96,7 @@ class ZvecAlgorithmTypedDict(ZvecHNSWTypedDict): @cli.command() @click_parameter_decorators_from_typed_dict(ZvecAlgorithmTypedDict) def Zvec(**parameters: Unpack[ZvecAlgorithmTypedDict]): + """Run Zvec using the selected dataset's distance metric.""" index_type = parameters["index_type"].lower() context = click.get_current_context() unsupported = ( @@ -117,11 +109,8 @@ def Zvec(**parameters: Unpack[ZvecAlgorithmTypedDict]): message = f"--{parameter.replace('_', '-')} does not apply to {index_type}" raise click.UsageError(message) - metric_type = parameters["metric_type"] - metric_type = MetricType(metric_type.upper()) if metric_type else None if index_type == "diskann": case_config = ZvecDiskANNIndexConfig( - metric_type=metric_type, max_degree=parameters["max_degree"], build_list_size=parameters["build_list_size"], search_list_size=parameters["search_list_size"], @@ -129,7 +118,6 @@ def Zvec(**parameters: Unpack[ZvecAlgorithmTypedDict]): ) else: case_config = ZvecHNSWIndexConfig( - metric_type=metric_type, M=parameters["m"], ef_construction=parameters["ef_construction"], ef_search=parameters["ef_search"], From 73c8be074db3158db06feca60cb0d55bd61d7bb0 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 24 Oct 2026 15:14:25 +0800 Subject: [PATCH 4/9] fix error for metric-type 1 Signed-off-by: Wayne Gao --- tests/test_zvec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_zvec.py b/tests/test_zvec.py index 1053663ce..32ffeaadd 100644 --- a/tests/test_zvec.py +++ b/tests/test_zvec.py @@ -344,7 +344,8 @@ def test_cli_rejects_metric_override(monkeypatch: pytest.MonkeyPatch, index_type ["--path", "unused", "--index-type", index_type, "--metric-type", metric_type, "--dry-run"], ) assert result.exit_code == 2, result.output - assert "No such option: --metric-type" in result.output + assert "No such option" in result.output + assert "--metric-type" in result.output run.assert_not_called() From 549a371a3663038d32aca289b93db6b6a8d3b36c Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 24 Oct 2026 17:59:54 +0800 Subject: [PATCH 5/9] fix(zvec): require SDK 0.7.0 or newer - Require zvec>=0.7.0 for Collection.close() support - Update setup instructions to install the zvec and test extras - Add regression coverage for the SDK dependency floor Signed-off-by: Wayne Gao --- README.md | 12 +++++++----- pyproject.toml | 2 +- tests/test_zvec.py | 9 +++++++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 36d32e78b..1b8602c5e 100644 --- a/README.md +++ b/README.md @@ -598,19 +598,21 @@ To list the options for zvec, execute vectordbbench zvec --help #### Zvec DiskANN - DiskANN requires a zvec SDK exposing `DiskAnnIndexParam`, `DiskAnnQueryParam`, and `Collection.close()`. + DiskANN requires `zvec>=0.7.0`, which provides `Collection.close()`. The SDK must also expose + `DiskAnnIndexParam` and `DiskAnnQueryParam`. This adapter uses zvec's native C++ DiskANN, not the Microsoft Rust implementation. - From this updated checkout on Linux, in an environment with benchmark dependencies and the DiskANN-enabled SDK - already installed, install the adapter and run its checks: + From this updated checkout on Linux, install the adapter with its Zvec and test dependencies, then run the checks: ```bash - python -m pip install --no-deps -e . - python -m pip install pytest + python -m pip install -e ".[zvec,test]" vectordbbench zvec --help python -m pytest tests/test_zvec.py -q ZVEC_NATIVE_TESTS=1 python -m pytest tests/test_zvec.py -k native_diskann -v ``` + The `zvec` extra enforces the SDK minimum. `--no-deps` bypasses dependency resolution; use it only after + independently verifying the installed SDK version and required APIs. + The native test builds 1,200 32-dimensional vectors in a temporary directory, checks insertion and index completion, closes/reopens the collection, searches it, and exercises rebuilding. It is a functionality test, not a recall or performance benchmark. It is skipped unless explicitly enabled on Linux. Run it successfully before a large benchmark. diff --git a/pyproject.toml b/pyproject.toml index 9fbbb7748..b3329cf44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ alisql = [ "mysqlclient" ] polardb = [ "PyMySQL" ] doris = [ "doris-vector-search" ] turbopuffer = [ "turbopuffer" ] -zvec = [ "zvec" ] +zvec = [ "zvec>=0.7.0" ] endee = [ "endee==0.1.10" ] lindorm = [ "opensearch-py" ] seekdb = [ "mysql-connector-python" ] diff --git a/tests/test_zvec.py b/tests/test_zvec.py index 32ffeaadd..9b29222fc 100644 --- a/tests/test_zvec.py +++ b/tests/test_zvec.py @@ -4,6 +4,7 @@ import pickle import random import sys +import tomllib from types import ModuleType, SimpleNamespace from unittest.mock import Mock @@ -17,6 +18,14 @@ from vectordb_bench.backend.filter import non_filter +def test_zvec_dependency_floor_requires_collection_close() -> None: + repo_root = Path(__file__).resolve().parents[1] + with (repo_root / "pyproject.toml").open("rb") as source: + dependencies = tomllib.load(source)["project"]["optional-dependencies"]["zvec"] + + assert "zvec>=0.7.0" in dependencies + + def test_diskann_config_and_registry_roundtrip(): config = ZvecDiskANNIndexConfig(metric_type=MetricType.COSINE, pq_chunk_num=96) assert config.index_param() == {"max_degree": 64, "list_size": 100, "pq_chunk_num": 96} From b75ebe7b0444eb49977e70935065285336c942d4 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 24 Oct 2026 20:49:18 +0800 Subject: [PATCH 6/9] feat(zvec): expose HNSW and DiskANN options in the run-test UI - Add algorithm-specific controls with HNSW as the default - Register Zvec build and search configuration inputs - Cover defaults, bounds, task generation and algorithm switching - Document UI usage and separate collection paths Signed-off-by: Wayne Gao --- README.md | 5 + tests/test_zvec_frontend.py | 174 ++++++++++++++++++ .../frontend/config/dbCaseConfigs.py | 101 ++++++++++ vectordb_bench/models.py | 5 + 4 files changed, 285 insertions(+) create mode 100644 tests/test_zvec_frontend.py diff --git a/README.md b/README.md index 1b8602c5e..17ab3de92 100644 --- a/README.md +++ b/README.md @@ -575,6 +575,11 @@ Options: HNSW remains the default. For both HNSW and DiskANN, the distance metric is determined by the selected dataset during task assembly, not by a Zvec CLI option. For example, `Performance768D10M` uses COSINE. +In the run-test UI, select Zvec and choose HNSW or DISKANN in the case settings. Only parameters for the selected algorithm +are displayed. Load cases expose build parameters; performance and streaming cases also expose search parameters. Defaults +match the CLI. Use separate collection paths for HNSW and DiskANN, and keep DiskANN build parameters unchanged for search-only +runs. + Existing HNSW commands are unchanged: ```bash diff --git a/tests/test_zvec_frontend.py b/tests/test_zvec_frontend.py new file mode 100644 index 000000000..7e6dc9394 --- /dev/null +++ b/tests/test_zvec_frontend.py @@ -0,0 +1,174 @@ +from unittest.mock import Mock + +import pytest + +from vectordb_bench.backend.cases import CaseLabel, CaseType +from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import IndexType +from vectordb_bench.backend.clients.zvec.config import ZvecConfig, ZvecDiskANNIndexConfig, ZvecHNSWIndexConfig +from vectordb_bench.frontend.components.run_test.caseSelector import dbCaseConfigSetting +from vectordb_bench.frontend.components.run_test.generateTasks import generate_tasks +from vectordb_bench.frontend.config.dbCaseConfigs import MAX_STREAMLIT_INT, UICaseItem +from vectordb_bench.models import CaseConfig, CaseConfigParamType, TaskConfig + + +@pytest.fixture( + params=[CaseType.CapacityDim128, CaseType.Performance768D10M, CaseType.StreamingPerformanceCase], + ids=["load", "performance", "streaming"], +) +def ui_case(request: pytest.FixtureRequest) -> CaseConfig: + return CaseConfig(case_id=request.param) + + +def render_zvec_ui( + case: CaseConfig, + values: dict[str, object] | None = None, + config: dict[CaseConfigParamType, object] | None = None, +) -> tuple[dict[CaseConfigParamType, object], dict[str, dict[str, object]]]: + values = {} if values is None else values + config = {} if config is None else config + inputs: dict[str, dict[str, object]] = {} + container = Mock() + container.columns.side_effect = lambda count: [container] * count + + def selectbox(_label, options, **kwargs): + name = kwargs["key"].rsplit("-", 1)[-1] + inputs[name] = {"options": options, **kwargs} + value = values.get(name, options[kwargs.get("index", 0)]) + assert value in options + return value + + def number_input(_label, **kwargs): + name = kwargs["key"].rsplit("-", 1)[-1] + inputs[name] = kwargs + value = values.get(name, kwargs["value"]) + assert isinstance(value, int) + assert kwargs["min_value"] <= value <= kwargs["max_value"] + return value + + container.selectbox.side_effect = selectbox + container.number_input.side_effect = number_input + item = UICaseItem(cases=[case], label="Zvec test", caseLabel=case.case.label) + dbCaseConfigSetting(container, {DB.Zvec: {item: config}}, item, [DB.Zvec]) + container.write.assert_not_called() + return config, inputs + + +def task_from_ui(case: CaseConfig, config: dict[CaseConfigParamType, object]) -> TaskConfig: + tasks = generate_tasks( + [DB.Zvec], + {DB.Zvec: ZvecConfig(db_label="ui-test", path="unused")}, + [case], + {DB.Zvec: {case: config}}, + batch_size=1, + ) + assert len(tasks) == 1 + return tasks[0] + + +def test_zvec_ui_defaults_to_hnsw(ui_case: CaseConfig) -> None: + config, inputs = render_zvec_ui(ui_case) + + assert inputs["IndexType"]["options"] == [IndexType.HNSW.value, IndexType.DISKANN.value] + assert config[CaseConfigParamType.IndexType] == IndexType.HNSW.value + expected_inputs = {"IndexType", "M", "ef_construction", "quantize_type"} + if ui_case.case.label != CaseLabel.Load: + expected_inputs.update({"ef_search", "is_using_refiner"}) + assert set(inputs) == expected_inputs + task = task_from_ui(ui_case, config) + assert isinstance(task.db_case_config, ZvecHNSWIndexConfig) + assert task.db_case_config == ZvecHNSWIndexConfig() + + +def test_zvec_ui_diskann_parameters_reach_task(ui_case: CaseConfig) -> None: + config, inputs = render_zvec_ui( + ui_case, + { + "IndexType": IndexType.DISKANN.value, + "max_degree": 32, + "build_list_size": 80, + "pq_chunk_num": 32, + "search_list_size": 600, + }, + ) + + expected_inputs = {"IndexType", "max_degree", "build_list_size", "pq_chunk_num"} + if ui_case.case.label != CaseLabel.Load: + expected_inputs.add("search_list_size") + assert set(inputs) == expected_inputs + task = task_from_ui(ui_case, config) + assert isinstance(task.db_case_config, ZvecDiskANNIndexConfig) + assert task.db_case_config == ZvecDiskANNIndexConfig( + max_degree=32, + build_list_size=80, + pq_chunk_num=32, + search_list_size=300 if ui_case.case.label == CaseLabel.Load else 600, + ) + + +@pytest.mark.parametrize("quantize_type", ["", "fp16", "int8"]) +def test_zvec_ui_hnsw_parameters_reach_task(quantize_type: str) -> None: + case = CaseConfig(case_id=CaseType.Performance768D10M) + config, _inputs = render_zvec_ui( + case, + { + "M": 32, + "ef_construction": 250, + "ef_search": 450, + "quantize_type": quantize_type, + "is_using_refiner": True, + }, + ) + + assert task_from_ui(case, config).db_case_config == ZvecHNSWIndexConfig( + M=32, + ef_construction=250, + ef_search=450, + quantize_type=quantize_type, + is_using_refiner=True, + ) + + +@pytest.mark.parametrize( + ("name", "minimum", "maximum", "default"), + [ + ("max_degree", 1, 100, 64), + ("build_list_size", 10, 100, 100), + ("search_list_size", 1, MAX_STREAMLIT_INT, 300), + ("pq_chunk_num", 0, 1024, 0), + ], +) +def test_zvec_ui_diskann_bounds_and_defaults(name: str, minimum: int, maximum: int, default: int) -> None: + case = CaseConfig(case_id=CaseType.Performance768D10M) + config, inputs = render_zvec_ui(case, {"IndexType": IndexType.DISKANN.value}) + + assert inputs[name]["min_value"] == minimum + assert inputs[name]["max_value"] == maximum + assert inputs[name]["value"] == default + assert task_from_ui(case, config).db_case_config == ZvecDiskANNIndexConfig() + + +def test_zvec_ui_algorithm_switch_ignores_hidden_parameters() -> None: + case = CaseConfig(case_id=CaseType.Performance768D10M) + config, _inputs = render_zvec_ui(case, {"M": 32, "quantize_type": "int8", "is_using_refiner": True}) + config, inputs = render_zvec_ui( + case, + {"IndexType": IndexType.DISKANN.value, "max_degree": 32, "pq_chunk_num": 48}, + config, + ) + + assert CaseConfigParamType.quantize_type in config + assert "quantize_type" not in inputs + diskann = task_from_ui(case, config).db_case_config + assert isinstance(diskann, ZvecDiskANNIndexConfig) + assert diskann == ZvecDiskANNIndexConfig(max_degree=32, pq_chunk_num=48) + assert "quantize_type" not in diskann.model_dump() + + config, inputs = render_zvec_ui(case, {"IndexType": IndexType.HNSW.value}, config) + + assert CaseConfigParamType.pq_chunk_num in config + assert "pq_chunk_num" not in inputs + hnsw = task_from_ui(case, config).db_case_config + assert isinstance(hnsw, ZvecHNSWIndexConfig) + assert hnsw == ZvecHNSWIndexConfig() + assert "pq_chunk_num" not in hnsw.model_dump() diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 678b53288..58f543a45 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -2865,6 +2865,86 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) +CaseConfigParamInput_IndexType_Zvec = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputType=InputType.Option, + inputConfig={"options": [IndexType.HNSW.value, IndexType.DISKANN.value]}, + inputHelp="Select the Zvec index algorithm. HNSW is the default; use separate paths for different algorithms.", +) + +CaseConfigParamInput_M_Zvec = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 50}, + inputHelp="HNSW graph parameter M.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, IndexType.HNSW.value) == IndexType.HNSW.value, +) + +CaseConfigParamInput_ef_construction_Zvec = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + inputType=InputType.Number, + inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 500}, + inputHelp="HNSW construction candidate list size.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, IndexType.HNSW.value) == IndexType.HNSW.value, +) + +CaseConfigParamInput_ef_search_Zvec = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputType=InputType.Number, + inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 300}, + inputHelp="HNSW query candidate list size.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, IndexType.HNSW.value) == IndexType.HNSW.value, +) + +CaseConfigParamInput_quantize_type_Zvec = CaseConfigInput( + label=CaseConfigParamType.quantize_type, + displayLabel="quantize_type (empty = FP32)", + inputType=InputType.Option, + inputConfig={"options": ["", "fp16", "int8"]}, + inputHelp="HNSW scalar quantization. Leave empty for unquantized FP32 vectors.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, IndexType.HNSW.value) == IndexType.HNSW.value, +) + +CaseConfigParamInput_is_using_refiner_Zvec = CaseConfigInput( + label=CaseConfigParamType.is_using_refiner, + inputType=InputType.Bool, + inputConfig={"value": False}, + inputHelp="Refine HNSW ef_search candidates to top-k with unquantized vectors.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, IndexType.HNSW.value) == IndexType.HNSW.value, +) + +CaseConfigParamInput_max_degree_Zvec = CaseConfigInput( + label=CaseConfigParamType.max_degree, + inputType=InputType.Number, + inputConfig={"min": 1, "max": 100, "value": 64}, + inputHelp="DiskANN graph out-degree.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType) == IndexType.DISKANN.value, +) + +CaseConfigParamInput_build_list_size_Zvec = CaseConfigInput( + label=CaseConfigParamType.build_list_size, + inputType=InputType.Number, + inputConfig={"min": 10, "max": 100, "value": 100}, + inputHelp="DiskANN construction candidate list size.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType) == IndexType.DISKANN.value, +) + +CaseConfigParamInput_search_list_size_Zvec = CaseConfigInput( + label=CaseConfigParamType.search_list_size, + inputType=InputType.Number, + inputConfig={"min": 1, "max": MAX_STREAMLIT_INT, "value": 300}, + inputHelp="DiskANN query candidate list size; the engine uses at least top-k.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType) == IndexType.DISKANN.value, +) + +CaseConfigParamInput_pq_chunk_num_Zvec = CaseConfigInput( + label=CaseConfigParamType.pq_chunk_num, + inputType=InputType.Number, + inputConfig={"min": 0, "max": 1024, "value": 0}, + inputHelp="DiskANN PQ bytes per vector. 0 selects automatically; nonzero must not exceed the vector dimension.", + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType) == IndexType.DISKANN.value, +) + CaseConfigParamInput_IndexType_Lindorm = CaseConfigInput( label=CaseConfigParamType.IndexType, inputHelp="Select Index Type", @@ -3066,6 +3146,23 @@ class FilterType(Enum): LanceDBPerformanceConfig = LanceDBLoadConfig +ZvecLoadConfig = [ + CaseConfigParamInput_IndexType_Zvec, + CaseConfigParamInput_M_Zvec, + CaseConfigParamInput_ef_construction_Zvec, + CaseConfigParamInput_quantize_type_Zvec, + CaseConfigParamInput_max_degree_Zvec, + CaseConfigParamInput_build_list_size_Zvec, + CaseConfigParamInput_pq_chunk_num_Zvec, +] + +ZvecPerformanceConfig = [ + *ZvecLoadConfig, + CaseConfigParamInput_ef_search_Zvec, + CaseConfigParamInput_is_using_refiner_Zvec, + CaseConfigParamInput_search_list_size_Zvec, +] + AWSOpensearchLoadingConfig = [ CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch, CaseConfigParamInput_ENGINE_NAME_AWSOpensearch, @@ -3489,6 +3586,10 @@ class FilterType(Enum): CaseLabel.Load: LanceDBLoadConfig, CaseLabel.Performance: LanceDBPerformanceConfig, }, + DB.Zvec: { + CaseLabel.Load: ZvecLoadConfig, + CaseLabel.Performance: ZvecPerformanceConfig, + }, DB.TencentElasticsearch: { CaseLabel.Load: TencentElasticsearchLoadingConfig, CaseLabel.Performance: TencentElasticsearchPerformanceConfig, diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index 21eea9e95..48adb4a8c 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -65,6 +65,8 @@ class CaseConfigParamType(Enum): EF = "ef" SearchList = "search_list" ef_search = "ef_search" + quantize_type = "quantize_type" + is_using_refiner = "is_using_refiner" Nlist = "nlist" Nprobe = "nprobe" MaxConnections = "maxConnections" @@ -110,6 +112,9 @@ class CaseConfigParamType(Enum): l_value_ib = "l_value_ib" l_value_is = "l_value_is" search_list_size = "search_list_size" + max_degree = "max_degree" + build_list_size = "build_list_size" + pq_chunk_num = "pq_chunk_num" max_alpha = "max_alpha" num_dimensions = "num_dimensions" num_bits_per_dimension = "num_bits_per_dimension" From b4bc5f9e0becc544522a087635b218ba5fadd86a Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Sat, 24 Oct 2026 21:09:13 +0800 Subject: [PATCH 7/9] docs(zvec): fix DiskANN README rendering Remove common leading indentation so the heading, parameter table and command examples render correctly. Preserve all documentation content and command continuation indentation. Signed-off-by: Wayne Gao --- README.md | 122 +++++++++++++++++++++++++++--------------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 17ab3de92..fac00616e 100644 --- a/README.md +++ b/README.md @@ -601,67 +601,67 @@ To list the options for zvec, execute vectordbbench zvec --help results ``` - #### Zvec DiskANN - - DiskANN requires `zvec>=0.7.0`, which provides `Collection.close()`. The SDK must also expose - `DiskAnnIndexParam` and `DiskAnnQueryParam`. - This adapter uses zvec's native C++ DiskANN, not the Microsoft Rust implementation. - From this updated checkout on Linux, install the adapter with its Zvec and test dependencies, then run the checks: - - ```bash - python -m pip install -e ".[zvec,test]" - vectordbbench zvec --help - python -m pytest tests/test_zvec.py -q - ZVEC_NATIVE_TESTS=1 python -m pytest tests/test_zvec.py -k native_diskann -v - ``` - - The `zvec` extra enforces the SDK minimum. `--no-deps` bypasses dependency resolution; use it only after - independently verifying the installed SDK version and required APIs. - - The native test builds 1,200 32-dimensional vectors in a temporary directory, checks insertion and index completion, - closes/reopens the collection, searches it, and exercises rebuilding. It is a functionality test, not a recall or - performance benchmark. It is skipped unless explicitly enabled on Linux. Run it successfully before a large benchmark. - - | Option | Default | Meaning | - | --- | --- | --- | - | `--index-type` | hnsw | Select hnsw or diskann | - | `--max-degree` | 64 | DiskANN graph degree, 1-100 | - | `--build-list-size` | 100 | Construction candidate list, 10-100 | - | `--search-list-size` | 300 | Query candidate list, at least 1; the engine uses at least top-k | - | `--pq-chunk-num` | 0 | PQ bytes per vector; 0 is automatic, otherwise at most min(dimension, 1024) | - - DiskANN stores FP32 vectors and uses internal PQ for graph traversal. HNSW options (`--m`, `--ef-construction`, - `--ef-search`, `--quantize-type`, `--is-using-refiner`) are rejected in DiskANN mode, and DiskANN options are rejected - in HNSW mode. PQ chunks are not equivalent to HNSW INT8 scalar quantization. - - After the checks pass, build a separate DiskANN collection. The load stage includes `optimize()`, which builds the - target index; flushing alone does not establish that a DiskANN index was built. `--drop-old` replaces any existing - DiskANN collection at this path. The adapter refuses to use or delete an existing non-DiskANN collection. - - ```bash - export DATASET_LOCAL_DIR="$PWD/dataset" - vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ - --case-type Performance768D10M \ - --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ - --drop-old --load --skip-search-serial --skip-search-concurrent - ``` - - Then search the same index without reloading or rebuilding it: - - ```bash - vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ - --case-type Performance768D10M \ - --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ - --search-list-size 300 --num-concurrency 12,14,16,18,20 \ - --skip-drop-old --skip-load --search-serial --search-concurrent - ``` - - Search-only runs validate the stored dimensions, metric, build parameters, positive document count and complete - indexing. Keep build parameters unchanged when reopening; sweep `--search-list-size` to compare QPS at matched recall. - These examples use the Cohere dataset's COSINE metric for both building and searching. For an HNSW/DiskANN - comparison, use the same dataset and separate collection paths; do not relabel an existing IP index as COSINE. - Multiple search processes can duplicate in-memory PQ codes and metadata, so measure total process memory as well - as QPS. +#### Zvec DiskANN + +DiskANN requires `zvec>=0.7.0`, which provides `Collection.close()`. The SDK must also expose +`DiskAnnIndexParam` and `DiskAnnQueryParam`. +This adapter uses zvec's native C++ DiskANN, not the Microsoft Rust implementation. +From this updated checkout on Linux, install the adapter with its Zvec and test dependencies, then run the checks: + +```bash +python -m pip install -e ".[zvec,test]" +vectordbbench zvec --help +python -m pytest tests/test_zvec.py -q +ZVEC_NATIVE_TESTS=1 python -m pytest tests/test_zvec.py -k native_diskann -v +``` + +The `zvec` extra enforces the SDK minimum. `--no-deps` bypasses dependency resolution; use it only after +independently verifying the installed SDK version and required APIs. + +The native test builds 1,200 32-dimensional vectors in a temporary directory, checks insertion and index completion, +closes/reopens the collection, searches it, and exercises rebuilding. It is a functionality test, not a recall or +performance benchmark. It is skipped unless explicitly enabled on Linux. Run it successfully before a large benchmark. + +| Option | Default | Meaning | +| --- | --- | --- | +| `--index-type` | hnsw | Select hnsw or diskann | +| `--max-degree` | 64 | DiskANN graph degree, 1-100 | +| `--build-list-size` | 100 | Construction candidate list, 10-100 | +| `--search-list-size` | 300 | Query candidate list, at least 1; the engine uses at least top-k | +| `--pq-chunk-num` | 0 | PQ bytes per vector; 0 is automatic, otherwise at most min(dimension, 1024) | + +DiskANN stores FP32 vectors and uses internal PQ for graph traversal. HNSW options (`--m`, `--ef-construction`, +`--ef-search`, `--quantize-type`, `--is-using-refiner`) are rejected in DiskANN mode, and DiskANN options are rejected +in HNSW mode. PQ chunks are not equivalent to HNSW INT8 scalar quantization. + +After the checks pass, build a separate DiskANN collection. The load stage includes `optimize()`, which builds the +target index; flushing alone does not establish that a DiskANN index was built. `--drop-old` replaces any existing +DiskANN collection at this path. The adapter refuses to use or delete an existing non-DiskANN collection. + +```bash +export DATASET_LOCAL_DIR="$PWD/dataset" +vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ + --case-type Performance768D10M \ + --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ + --drop-old --load --skip-search-serial --skip-search-concurrent +``` + +Then search the same index without reloading or rebuilding it: + +```bash +vectordbbench zvec --index-type diskann --path Performance768D10M_DiskANN \ + --case-type Performance768D10M \ + --max-degree 64 --build-list-size 100 --pq-chunk-num 96 \ + --search-list-size 300 --num-concurrency 12,14,16,18,20 \ + --skip-drop-old --skip-load --search-serial --search-concurrent +``` + +Search-only runs validate the stored dimensions, metric, build parameters, positive document count and complete +indexing. Keep build parameters unchanged when reopening; sweep `--search-list-size` to compare QPS at matched recall. +These examples use the Cohere dataset's COSINE metric for both building and searching. For an HNSW/DiskANN +comparison, use the same dataset and separate collection paths; do not relabel an existing IP index as COSINE. +Multiple search processes can duplicate in-memory PQ codes and metadata, so measure total process memory as well +as QPS. ### Run Doris from command line From c41450cd45a3501266b6c740f0692c6f5cfbb86d Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Tue, 27 Oct 2026 20:35:19 +0800 Subject: [PATCH 8/9] fix(vespa): follow Ruff datetime import convention (#892) Signed-off-by: jamesgao-jpg (cherry picked from commit 7e6ea9b78532f4cea64a0e8cf0a49424b084049b) --- vectordb_bench/backend/clients/vespa/vespa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordb_bench/backend/clients/vespa/vespa.py b/vectordb_bench/backend/clients/vespa/vespa.py index 12fa65d5e..99a4e46c4 100644 --- a/vectordb_bench/backend/clients/vespa/vespa.py +++ b/vectordb_bench/backend/clients/vespa/vespa.py @@ -1,4 +1,4 @@ -import datetime +import datetime as dt import json import logging import math @@ -490,7 +490,7 @@ def _create_application_package(self): ValidationID, ) - tomorrow = datetime.date.today() + datetime.timedelta(days=1) + tomorrow = dt.date.today() + dt.timedelta(days=1) if self._is_fts: fields = [ From 7f9772f06bd5d116ce801790c3c0f232bc74f3b0 Mon Sep 17 00:00:00 2001 From: Wayne Gao Date: Fri, 6 Nov 2026 22:48:29 +0800 Subject: [PATCH 9/9] fix(zvec): restore configuration delimiters after merge Restore the closing list bracket and Zvec mapping delimiter lost in b17574fe so frontend configuration parses again. --- vectordb_bench/frontend/config/dbCaseConfigs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index 0fb71348d..0c87bee90 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -3376,6 +3376,8 @@ class FilterType(Enum): CaseConfigParamInput_ef_search_Zvec, CaseConfigParamInput_is_using_refiner_Zvec, CaseConfigParamInput_search_list_size_Zvec, +] + KiviDBLoadConfig = [ CaseConfigParamInput_IndexType_KiviDB, CaseConfigParamInput_M_KiviDB, @@ -3875,6 +3877,7 @@ class FilterType(Enum): DB.Zvec: { CaseLabel.Load: ZvecLoadConfig, CaseLabel.Performance: ZvecPerformanceConfig, + }, DB.QdrantLocal: { CaseLabel.Load: QdrantLocalLoadConfig, CaseLabel.Performance: QdrantLocalPerformanceConfig,