zarr-python: Sharded array is very slow to load

Zarr version

‘2.13.7.dev9’

Numcodecs version

0.11.0

Python Version

3.10.9

Operating System

Linux

Installation

Pip from main branch

Description

Loading data using the new ShardingStorageTransformer is very slow.

I have a dataset of 6TB which I want to sample from randomly and I’ve experienced speed issues which I’m guessing is from the large number of files when the chunk size is small. The new sharding transformer could potentially solve my problem but so far it is even slower.

Output from the example attached

Type: minimal. Time: 0.737 s
Type: medium. Time: 5.382 s
Type: large. Time: 47.864 s
Type: sharded. Time: 76.932 s

Steps to reproduce

import os

os.environ["ZARR_V3_EXPERIMENTAL_API"] = "1"
os.environ["ZARR_V3_SHARDING"] = "1"

import numpy as np
import zarr
from zarr._storage.v3_storage_transformers import (
    ShardingStorageTransformer,
)
from time import perf_counter
from tempfile import TemporaryDirectory

with TemporaryDirectory() as tmp_dir:
    # %%
    # Create data stores
    shape = (100, 1000, 1000)
    data = np.random.random(shape)
    store = zarr.DirectoryStoreV3(f"{tmp_dir}/example.zarr")
    zgroup = zarr.group(store, overwrite=True, zarr_version=3)

    # Small chunks
    zarr_minimal_chunks = zarr.create(
        shape, chunks=(1, 1000, 1000), path="minimal", zarr_version=3, store=store
    )
    zarr_minimal_chunks[:] = data

    # Medium chunks
    zarr_medium_chunks = zarr.create(
        shape, chunks=(10, 1000, 1000), path="medium", zarr_version=3, store=store
    )
    zarr_medium_chunks[:] = data

    # Large chunks
    zarr_large_chunks = zarr.create(
        shape, chunks=(100, 1000, 1000), path="large", zarr_version=3, store=store
    )
    zarr_large_chunks[:] = data

    # Large sharded chunks
    zarr_sharded_chunks = zarr.create(
        shape,
        chunks=(100, 1000, 1000),
        path="sharded",
        zarr_version=3,
        storage_transformers=[
            ShardingStorageTransformer(
                "indexed",
                chunks_per_shard=(100, 1, 1),
            )
        ],
        store=store,
    )
    zarr_sharded_chunks[:] = data

    # Close store to ensure data is flushed to disk? Probably unnecessary
    store.close()

    def zarr_load_steps(array: zarr.Array, name: str, num_steps: int):
        start = perf_counter()
        for i in range(num_steps):
            array[i]

        print(f"Type: {name}. Time: {perf_counter() - start:.3f} s")

    for name, f in (
        ("minimal", zarr_minimal_chunks),
        ("medium", zarr_medium_chunks),
        ("large", zarr_large_chunks),
        ("sharded", zarr_sharded_chunks),
    ):
        zarr_load_steps(f, name, 100)

Additional output

No response

About this issue

  • Original URL
  • State: closed
  • Created 4 years ago
  • Comments: 15 (12 by maintainers)

Most upvoted comments

👋 @rbavery!

The example above was using an ancient version of sharding… We’ve come a long way!

I adapted the original example:

import time
from tempfile import TemporaryDirectory
import numpy as np
import zarr
from zarr.storage.logging import LoggingStore
from zarr.codecs import ShardingCodec, TransposeCodec, BytesCodec, BloscCodec


class Timer:
    def __init__(self, hint: str):
        self.hint = hint
    
    def __enter__(self):
        self.start_time = time.time()  # Record the start time
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.end_time = time.time()  # Record the end time
        self.duration = self.end_time - self.start_time  # Calculate the duration
        print(f"{self.hint}: {self.duration:.4f} seconds")


array_kwargs = {
    "minimal": {'chunks': (1, 1000, 1000)},
    "medium": {'chunks': (10, 1000, 1000)},
    "large": {'chunks': (100, 1000, 1000)},
    "sharded": {
        'chunks': (100, 1000, 1000),  # same as minimal
        'codecs': [ShardingCodec(
            chunk_shape=(1, 1000, 1000),
            codecs=[BytesCodec(),],
            index_location="start",
            )],
    }
}


with TemporaryDirectory() as tmp_dir:

    store = zarr.storage.LocalStore(tmp_dir, mode='w')
    # store = LoggingStore(store)
    shape = (100, 1000, 1000)
    data = np.random.random(shape)

    zgroup = zarr.group(store, zarr_format=3)

    for path, kwargs in array_kwargs.items():

        with Timer(f"initialize {path}"):
            arr = zarr.create(
                shape, path=path, zarr_format=3, store=store, **kwargs
            )
        with Timer(f"write {path}"):
            arr[:] = data

        with Timer(f"read {path}"):
            _ = arr[:]

# initialize minimal: 0.0036 seconds
# write minimal: 0.7344 seconds
# read minimal: 0.4661 seconds
# initialize medium: 0.0011 seconds
# write medium: 0.9915 seconds
# read medium: 0.7139 seconds
# initialize large: 0.0014 seconds
# write large: 1.2540 seconds
# read large: 0.5786 seconds
# initialize sharded: 0.0045 seconds
# write sharded: 14.0472 seconds
# read sharded: 0.4677 seconds

☝️ based on this, the “slow to load” issue that opened this ticket seems to be gone (reading shards is almost exactly the same speed as reading minimal)

Here is the updated code for zarr-python 3.0.0b0:

import numpy as np
import zarr

from time import perf_counter
from tempfile import TemporaryDirectory

import zarr.codecs
import zarr.storage

with TemporaryDirectory() as tmp_dir:
    # %%
    # Create data stores
    shape = (100, 1000, 1000)
    data = np.random.random(shape)
    store = zarr.storage.LocalStore(tmp_dir, mode="a")

    # Small chunks
    zarr_minimal_chunks = zarr.open(
        store=store, path="minimal.zarr", shape=shape, dtype=data.dtype, chunk_shape=(1, 1000, 1000)
    )
    zarr_minimal_chunks[:] = data

    # Medium chunks
    zarr_medium_chunks = zarr.open(
        store=store, path="medium.zarr", shape=shape, dtype=data.dtype, chunk_shape=(10, 1000, 1000)
    )
    zarr_medium_chunks[:] = data

    # Large chunks
    zarr_large_chunks = zarr.open(
        store=store, path="large.zarr", shape=shape, dtype=data.dtype, chunk_shape=(100, 1000, 1000)
    )
    zarr_large_chunks[:] = data

    # Large sharded chunks
    zarr_sharded_chunks = zarr.open(
        store=store,
        path="sharded.zarr",
        shape=shape,
        chunk_shape=(100, 1000, 1000),
        dtype=data.dtype,
        codecs=[
            zarr.codecs.ShardingCodec(
                chunk_shape=(1, 1000, 1000), codecs=[zarr.codecs.BytesCodec()]
            )
        ],
    )
    zarr_sharded_chunks[:] = data

    def zarr_load_steps(array: zarr.Array, name: str, num_steps: int):
        start = perf_counter()
        for i in range(num_steps):
            array[i]

        print(f"Type: {name}. Time: {perf_counter() - start:.3f} s")

    for name, f in (
        ("minimal", zarr_minimal_chunks),
        ("medium", zarr_medium_chunks),
        ("large", zarr_large_chunks),
        ("sharded", zarr_sharded_chunks),
    ):
        zarr_load_steps(f, name, 100)

I am getting the following numbers. They seem reasonable to me.

Type: minimal. Time: 0.390 s
Type: medium. Time: 1.387 s
Type: large. Time: 12.539 s
Type: sharded. Time: 0.553 s