qdrant: Optimization error: Out of memory

Current Behavior

I tried to load 50 millions vectors in qdrant v1.3.2 with hnsw.on_disk = true and vectors.on_disk = true qdrant v1.3.2 run without docker

Steps to Reproduce

1. create index

m =48 ef_construct = 256 shard_number = 4 replication_factor = 1 on_disk_payload = true hnsw.on_disk = true quantization = “Scalar” vectors.on_disk = true memmap_threshold = 20000 indexing_threshold = 20000 max_segment_size = 2500000

UPD: I used 1 server from cluster.

1. upload a batch
2. insert requests are working
3. indexation are not working

collection info

{
    "result": {
        "status": "red",
        "optimizer_status": {
            "error": "Out of memory, free: 9223372034152468480, IO Error: Cannot allocate memory (os error 12)"
        },
        "vectors_count": 26017604,
        "indexed_vectors_count": 123717,
        "points_count": 26017604,
        "segments_count": 29,
        "config": {
            "params": {
                "vectors": {
                    "size": 512,
                    "distance": "Cosine",
                    "on_disk": true
                },
                "shard_number": 4,
                "replication_factor": 1,
                "write_consistency_factor": 1,
                "on_disk_payload": true
            },
            "hnsw_config": {
                "m": 48,
                "ef_construct": 256,
                "full_scan_threshold": 10000,
                "max_indexing_threads": 0,
                "on_disk": true
            },
            "optimizer_config": {
                "deleted_threshold": 0.2,
                "vacuum_min_vector_number": 1000,
                "default_segment_number": 0,
                "max_segment_size": 2500000,
                "memmap_threshold": 20000,
                "indexing_threshold": 20000,
                "flush_interval_sec": 5,
                "max_optimization_threads": 1
            },
            "wal_config": {
                "wal_capacity_mb": 32,
                "wal_segments_ahead": 0
            },
            "quantization_config": {
                "scalar": {
                    "type": "int8",
                    "quantile": 0.99,
                    "always_ram": true
                }
            }
        },
        "payload_schema": {
            "field1": {
                "data_type": "integer",
                "points": 259848
            },
            "field2": {
                "data_type": "text",
                "params": {
                    "type": "text",
                    "tokenizer": "word",
                    "min_token_len": 2,
                    "max_token_len": 42,
                    "lowercase": false
                },
                "points": 227266
            },
            "field3": {
                "data_type": "keyword",
                "points": 303773
            },
            "string-field": {
                "data_type": "keyword",
                "points": 303773
            }
        }
    },
    "status": "ok",
    "time": 0.000286557
}

log messages

Jul 28 08:05:17  [2023-07-28T05:05:17.232Z ERROR qdrant::startup] Panic occurred in file lib/collection/src/update_handler.rs at line 246: Optimization error: Out of memory, free: 9223372033790472192, IO Error: Cannot all>
Jul 28 08:05:17      
Jul 28 08:05:17        14: clone
Jul 28 08:05:17        13: start_thread
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/sys/unix/thread.rs:108:17
Jul 28 08:05:17        12: std::sys::unix::thread::Thread::new::thread_start
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/alloc/src/boxed.rs:1985:9
Jul 28 08:05:17        11: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/alloc/src/boxed.rs:1985:9
Jul 28 08:05:17        10: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
Jul 28 08:05:17         9: core::ops::function::FnOnce::call_once{{vtable.shim}}
Jul 28 08:05:17         8: std::sys_common::backtrace::__rust_begin_short_backtrace
Jul 28 08:05:17         7: tokio::runtime::task::raw::poll
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/core/src/panicking.rs:67:14
Jul 28 08:05:17         6: core::panicking::panic_fmt
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs:593:5
Jul 28 08:05:17         5: rust_begin_unwind
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/sys_common/backtrace.rs:151:18
Jul 28 08:05:17         4: std::sys_common::backtrace::__rust_end_short_backtrace
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs:597:13
Jul 28 08:05:17         3: std::panicking::begin_panic_handler::{{closure}}
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs:709:13
Jul 28 08:05:17         2: std::panicking::rust_panic_with_hook
Jul 28 08:05:17                   at /rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/alloc/src/boxed.rs:1999:9
Jul 28 08:05:17         1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
Jul 28 08:05:17         0: qdrant::startup::setup_panic_hook::{{closure}}
Jul 28 08:05:17  [2023-07-28T05:05:17.232Z ERROR qdrant::startup] Panic backtrace:

I stopped ~ 16 millions vectors.

I updated at commit https://github.com/qdrant/qdrant/commit/7164ac4a5987d28f1c93f5712aef8e09e7d93555 yesterday.