Commit ⋅ activeloopai/deeplake

@@ -94,6 +94,7 @@

Loading

from deeplake.core.sample import Sample
from itertools import chain, repeat
from collections.abc import Iterable
from PIL import Image  # type: ignore


class ChunkEngine:

@@ -1481,7 +1482,8 @@

Loading

        cast: bool = True,
        copy: bool = False,
        decompress: bool = True,
    ) -> np.ndarray:
        to_pil: bool = False,
    ) -> Union[np.ndarray, Image.Image]:
        enc = self.chunk_id_encoder
        if self.is_fixed_shape and self.sample_compression is None:
            num_samples_per_chunk = self.num_samples_per_chunk

@@ -1490,6 +1492,16 @@

Loading

            local_sample_index = enc.translate_index_relative_to_chunks(
                global_sample_index
            )
        if to_pil:
            assert isinstance(chunk, SampleCompressedChunk)
            return chunk.read_sample(
                local_sample_index,
                cast=cast,
                copy=copy,
                decompress=decompress,
                to_pil=True,
            )

        return chunk.read_sample(
            local_sample_index, cast=cast, copy=copy, decompress=decompress
        )

@@ -18,7 +18,8 @@

Loading

from deeplake.util.link import get_path_creds_key, save_link_creds
from deeplake.util.video import normalize_index
import numpy as np
from typing import Optional, Dict, Any, Tuple
from typing import Optional, Dict, Any, Tuple, Union
from PIL import Image  # type: ignore


def retry_refresh_managed_creds(fn):

@@ -239,7 +240,8 @@

Loading

        cast: bool = True,
        copy: bool = False,
        decompress: bool = True,
    ) -> np.ndarray:
        to_pil: bool = False,
    ) -> Union[np.ndarray, Image.Image]:
        enc = self.chunk_id_encoder
        local_sample_index = enc.translate_index_relative_to_chunks(global_sample_index)
        sample_path = chunk.read_sample(

@@ -251,7 +253,12 @@

Loading

            return self.get_empty_sample()
        sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
        sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
        return read_linked_sample(sample_path, sample_creds_key, self.link_creds, False)
        read_sample = read_linked_sample(
            sample_path, sample_creds_key, self.link_creds, False
        )
        if to_pil:
            return read_sample.pil
        return read_sample.array

    def check_link_ready(self):
        missing_keys = self.link_creds.missing_keys

@@ -1,4 +1,4 @@

Loading

1		-	from typing import Iterable, Optional, Sequence, List, Union
	1	+	from typing import Iterable, Optional, Sequence, List, Union, Dict
2	2		from deeplake.constants import MB
3	3		from deeplake.integrations.pytorch.common import PytorchTransformFunction
4	4

@@ -31,6 +31,7 @@

Loading


import numpy as np
import deeplake
from PIL import Image  # type: ignore


mp = torch.multiprocessing.get_context()

@@ -64,6 +65,9 @@

Loading

def copy_tensor(x):
    if isinstance(x, Sample):
        x = x.array
    if isinstance(x, Image.Image):
        return x

    try:
        copy = cast_type(x)
    except AttributeError:

@@ -402,23 +406,26 @@

Loading

    def __init__(
        self,
        dataset,
        tensors: Sequence[str],
        use_local_cache: bool = False,
        tensors: Optional[Sequence[str]] = None,
        tobytes: Union[bool, Sequence[str]] = False,
        transform: Optional[PytorchTransformFunction] = PytorchTransformFunction(),
        num_workers: int = 1,
        shuffle: bool = False,
        buffer_size: int = 0,
        return_index: bool = True,
        pad_tensors: bool = False,
        decode_method: Optional[Dict[str, str]] = None,
    ) -> None:
        super().__init__()

        self.dataset = dataset
        self.transform = transform
        self.tensors = tensors
        self.tobytes = tobytes
        self.shuffle: bool = shuffle
        self.buffer_size: int = buffer_size * MB
        self.return_index: bool = return_index
        self.pad_tensors = pad_tensors
        self.decode_method = decode_method

        self.use_local_cache = use_local_cache
        self.scheduler = use_scheduler(num_workers, shuffle)

@@ -432,19 +439,15 @@

Loading

        streaming = SampleStreaming(
            dataset,
            tensors=self.tensors,  # type: ignore
            tobytes=self.tobytes,
            use_local_cache=use_local_cache,
            pad_tensors=self.pad_tensors,
            decode_method=self.decode_method,
        )

        self.schedules: List[Schedule] = self.scheduler.schedule(
            streaming.list_blocks()
        )

        self.shuffle: bool = shuffle
        self.buffer_size: int = buffer_size * MB
        self.return_index: bool = return_index

    def __iter__(self):
        worker_info = torch.utils.data.get_worker_info()
        schedule: Schedule = self.schedules[0]

@@ -455,10 +458,10 @@

Loading

        streaming = SampleStreaming(
            self.dataset,
            tensors=self.tensors,
            tobytes=self.tobytes,
            use_local_cache=self.use_local_cache,
            return_index=self.return_index,
            pad_tensors=self.pad_tensors,
            decode_method=self.decode_method,
        )

        if self.shuffle:

@@ -477,28 +480,28 @@

Loading

    def __init__(
        self,
        dataset,
        tensors: Sequence[str],
        use_local_cache: bool = False,
        tensors: Optional[Sequence[str]] = None,
        tobytes: Union[bool, Sequence[str]] = False,
        transform: PytorchTransformFunction = PytorchTransformFunction(),
        num_workers: int = 1,
        buffer_size: int = 512,
        batch_size: int = 1,
        return_index: bool = True,
        pad_tensors: bool = False,
        decode_method: Optional[Dict[str, str]] = None,
    ) -> None:
        super().__init__()

        self.torch_datset = TorchDataset(
            dataset,
            use_local_cache,
            tensors,
            tobytes,
            tensors=tensors,
            use_local_cache=use_local_cache,
            transform=None if buffer_size else transform,
            num_workers=num_workers,
            shuffle=True,
            return_index=return_index,
            pad_tensors=pad_tensors,
            decode_method=decode_method,
        )
        if buffer_size:
            self.transform = transform

@@ -13,6 +13,7 @@

Loading

from deeplake.constants import KB

from deeplake.tests.dataset_fixtures import enabled_non_gdrive_datasets
from PIL import Image  # type: ignore

try:
    from torch.utils.data._utils.collate import default_collate

@@ -639,10 +640,14 @@

Loading

    assert s == sum(list(range(254)))


def identity(x):
    return x


@requires_torch
@enabled_non_gdrive_datasets
@pytest.mark.parametrize("compression", [None, "jpeg"])
def test_pytorch_tobytes(ds, compressed_image_paths, compression):
def test_pytorch_decode(ds, compressed_image_paths, compression):
    with ds:
        ds.create_tensor("image", sample_compression=compression)
        ds.image.extend(

@@ -654,7 +659,7 @@

Loading

            ds.pytorch()
        return

    for i, batch in enumerate(ds.pytorch(tobytes=["image"])):
    for i, batch in enumerate(ds.pytorch(decode_method={"image": "tobytes"})):
        image = batch["image"][0]
        assert isinstance(image, bytes)
        if i < 5 and not compression:

@@ -666,6 +671,19 @@

Loading

            with open(compressed_image_paths["jpeg"][0], "rb") as f:
                assert f.read() == image

    if compression:
        ptds = ds.pytorch(decode_method={"image": "pil"}, collate_fn=identity)
        for i, batch in enumerate(ptds):
            image = batch[0]["image"]
            assert isinstance(image, Image.Image)
            if i < 5:
                np.testing.assert_array_equal(
                    np.array(image), i * np.ones((10, 10, 3), dtype=np.uint8)
                )
            elif i >= 5:
                with Image.open(compressed_image_paths["jpeg"][0]) as f:
                    np.testing.assert_array_equal(np.array(f), np.array(image))


@requires_torch
def test_rename(local_ds):

@@ -25,5 +25,8 @@

Loading

    assert arr.shape == (900, 900, 3)
    assert arr.dtype == np.uint32

    pil_img = flower.pil
    assert pil_img.size == (464, 513)


# TODO: test creating Sample with np.ndarray

@@ -1,6 +1,4 @@

Loading

import pickle
from deeplake.experimental import dataloader

import deeplake
import numpy as np
import pytest

@@ -14,6 +12,7 @@

Loading

from deeplake.constants import KB

from deeplake.tests.dataset_fixtures import enabled_non_gdrive_datasets
from PIL import Image  # type: ignore

try:
    from torch.utils.data._utils.collate import default_collate

@@ -63,9 +62,9 @@

Loading


    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return
    dl = dataloader(ds).batch(1).pytorch(num_workers=2)
    dl = ds.dataloader().batch(1).pytorch(num_workers=2)

    assert len(dl.dataset) == 16


@@ -79,7 +78,7 @@

Loading

            )

    sub_ds = ds[5:]
    sub_dl = dataloader(sub_ds).pytorch(num_workers=0)
    sub_dl = sub_ds.dataloader().pytorch(num_workers=0)

    for i, batch in enumerate(sub_dl):
        np.testing.assert_array_equal(

@@ -90,7 +89,7 @@

Loading

        )

    sub_ds2 = ds[8:12]
    sub_dl2 = dataloader(sub_ds2).pytorch(num_workers=0)
    sub_dl2 = sub_ds2.dataloader().pytorch(num_workers=0)

    for _ in range(2):
        for i, batch in enumerate(sub_dl2):

@@ -102,7 +101,7 @@

Loading

            )

    sub_ds3 = ds[:5]
    sub_dl3 = dataloader(sub_ds3).pytorch(num_workers=0)
    sub_dl3 = sub_ds3.dataloader().pytorch(num_workers=0)

    for _ in range(2):
        for i, batch in enumerate(sub_dl3):

@@ -127,11 +126,11 @@

Loading


    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return

    dl = (
        dataloader(ds)
        ds.dataloader()
        .batch(1)
        .transform(to_tuple, t1="image", t2="image2")
        .pytorch(num_workers=2)

@@ -161,10 +160,10 @@

Loading


    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return

    dl = dataloader(ds).transform({"image": double, "image2": None}).pytorch()
    dl = ds.dataloader().transform({"image": double, "image2": None}).pytorch()

    assert len(dl.dataset) == 16


@@ -209,10 +208,10 @@

Loading


    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return

    dl = dataloader(ds).pytorch(num_workers=0)
    dl = ds.dataloader().pytorch(num_workers=0)

    for _ in range(2):
        for batch in dl:

@@ -234,13 +233,13 @@

Loading


    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return

    with pytest.raises(TensorDoesNotExistError):
        dl = dataloader(ds).pytorch(tensors=["c", "d", "e"])
        dl = ds.dataloader().pytorch(tensors=["c", "d", "e"])

    dl = dataloader(ds).pytorch(tensors=["c", "d", "a"], return_index=False)
    dl = ds.dataloader().pytorch(tensors=["c", "d", "a"], return_index=False)

    for i, batch in enumerate(dl):
        c1, d1, a1 = batch

@@ -280,7 +279,7 @@

Loading

    base_storage.enable_readonly()
    ds = Dataset(storage=local_ds.storage, read_only=True, verbose=False)

    ptds = dataloader(ds).pytorch(num_workers=2)
    ptds = ds.dataloader().pytorch(num_workers=2)
    # no need to check input, only care that readonly works
    for _ in ptds:
        pass

@@ -313,7 +312,7 @@

Loading

            local_ds.images.pngs.flowers.append(img2)

    another_ds = deeplake.dataset(local_ds.path)
    dl = dataloader(another_ds).pytorch(return_index=False)
    dl = another_ds.dataloader().pytorch(return_index=False)
    for i, (cat, flower) in enumerate(dl):
        assert cat[0].shape == another_ds.images.jpegs.cats[i].numpy().shape
        assert flower[0].shape == another_ds.images.pngs.flowers[i].numpy().shape

@@ -326,7 +325,7 @@

Loading

        local_ds.create_tensor("strings", htype="text")
        local_ds.strings.extend([f"string{idx}" for idx in range(5)])

    ptds = dataloader(local_ds).pytorch()
    ptds = local_ds.dataloader().pytorch()
    for idx, batch in enumerate(ptds):
        np.testing.assert_array_equal(batch["strings"], f"string{idx}")


@@ -365,7 +364,7 @@

Loading

        ds.img2.extend(arr_list_2)
        ds.label.extend(label_list)

    ptds = dataloader(local_ds[index]).pytorch()
    ptds = local_ds[index].dataloader().pytorch()
    idxs = list(IndexEntry(index).indices(len(local_ds)))
    for idx, batch in enumerate(ptds):
        idx = idxs[idx]

@@ -387,7 +386,7 @@

Loading

            local_ds.b.append(1)
            local_ds.c.append(2)

    ptds = dataloader(local_ds).batch(4).pytorch(collate_fn=reorder_collate)
    ptds = local_ds.dataloader().batch(4).pytorch(collate_fn=reorder_collate)
    if shuffle:
        ptds = ptds.shuffle()
    for batch in ptds:

@@ -412,7 +411,7 @@

Loading

            local_ds.c.append(2 * np.ones((300, 300)))

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .pytorch(
            collate_fn=my_transform_collate,

@@ -439,7 +438,7 @@

Loading

@requires_libdeeplake
@enabled_non_gdrive_datasets
@pytest.mark.parametrize("compression", [None, "jpeg"])
def test_pytorch_tobytes(ds, compressed_image_paths, compression):
def test_pytorch_decode(ds, compressed_image_paths, compression):
    with ds:
        ds.create_tensor("image", sample_compression=compression)
        ds.image.extend(

@@ -448,10 +447,10 @@

Loading

        ds.image.extend([deeplake.read(compressed_image_paths["jpeg"][0])] * 5)
    if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
        with pytest.raises(ValueError):
            dl = dataloader(ds)
            dl = ds.dataloader()
        return

    ptds = dataloader(ds).pytorch(tobytes=["image"])
    ptds = ds.dataloader().pytorch(decode_method={"image": "tobytes"})

    for i, batch in enumerate(ptds):
        image = batch["image"][0]

@@ -465,6 +464,19 @@

Loading

            with open(compressed_image_paths["jpeg"][0], "rb") as f:
                assert f.read() == image

    if compression:
        ptds = ds.dataloader().numpy(decode_method={"image": "pil"})
        for i, batch in enumerate(ptds):
            image = batch[0]["image"]
            assert isinstance(image, Image.Image)
            if i < 5:
                np.testing.assert_array_equal(
                    np.array(image), i * np.ones((10, 10, 3), dtype=np.uint8)
                )
            elif i >= 5:
                with Image.open(compressed_image_paths["jpeg"][0]) as f:
                    np.testing.assert_array_equal(np.array(f), np.array(image))


@requires_torch
@requires_libdeeplake

@@ -476,7 +488,7 @@

Loading

        ds.rename_tensor("abc", "xyz")
        ds.rename_group("blue", "red")
        ds["red/green"].append([1, 2, 3, 4])
    loader = dataloader(ds).pytorch(return_index=False)
    loader = ds.dataloader().pytorch(return_index=False)
    for sample in loader:
        assert set(sample.keys()) == {"xyz", "red/green"}
        np.testing.assert_array_equal(np.array(sample["xyz"]), np.array([[1, 2, 3]]))

@@ -489,7 +501,7 @@

Loading

@requires_libdeeplake
def test_expiration_date_casting_to_string():
    ds = deeplake.dataset("hub://activeloop/cifar100-train")[0:10:2]
    loader = dataloader(ds).pytorch(return_index=False)
    loader = ds.dataloader().pytorch(return_index=False)
    for _ in loader:
        pass


@@ -505,7 +517,7 @@

Loading

            ds.xyz.append(i * np.ones((2, 2)))

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .pytorch(num_workers=num_workers, return_index=True)
    )

@@ -529,7 +541,7 @@

Loading

            ds.xyz.append(i * np.ones((2, 2)))

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .transform(index_transform)
        .pytorch(num_workers=num_workers, return_index=True)

@@ -557,7 +569,7 @@

Loading

            ds.xyz.append(i * np.ones((2, 2)))

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .transform({"xyz": double, "index": None})
        .pytorch(num_workers=num_workers, return_index=True)

@@ -571,7 +583,7 @@

Loading

            np.testing.assert_array_equal(2 * batch["index"][i], batch["xyz"][i][0, 0])

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .transform({"xyz": double})
        .pytorch(num_workers=num_workers, return_index=True)

@@ -595,7 +607,7 @@

Loading


    with pytest.raises(ValueError):
        ptds = (
            dataloader(local_ds)
            local_ds.dataloader()
            .batch(4)
            .pytorch(
                num_workers=num_workers, return_index=True, tensors=["xyz", "index"]

@@ -603,7 +615,7 @@

Loading

        )

    ptds = (
        dataloader(local_ds)
        local_ds.dataloader()
        .batch(4)
        .pytorch(num_workers=num_workers, return_index=True, tensors=["xyz"])
    )

@@ -622,7 +634,7 @@

Loading

        ds.create_tensor("y")
        ds.x.extend(list(range(5)))
        ds.y.extend(list(range(10)))
    ptds = dataloader(ds).pytorch()
    ptds = ds.dataloader().pytorch()
    for i, batch in enumerate(ptds):
        x, y = np.array(batch["x"][0]), np.array(batch["y"][0])
        np.testing.assert_equal(x, i)

@@ -637,16 +649,16 @@

Loading

        ds.create_tensor("y")
        ds.x.extend(list(range(5)))

    ptds = dataloader(ds).pytorch()
    ptds = ds.dataloader().pytorch()
    with pytest.raises(EmptyTensorError):
        for _ in ptds:
            pass

    ptds = dataloader(ds).pytorch(tensors=["x", "y"])
    ptds = ds.dataloader().pytorch(tensors=["x", "y"])
    with pytest.raises(EmptyTensorError):
        for _ in ptds:
            pass

    ptds = dataloader(ds).pytorch(tensors=["x"])
    ptds = ds.dataloader().pytorch(tensors=["x"])
    for _ in ptds:
        pass
imilarity index 100%
ename from deeplake/experimental/test_query.py
ename to deeplake/enterprise/test_query.py
imilarity index 93%
ename from deeplake/experimental/util.py
ename to deeplake/enterprise/util.py

@@ -1,17 +1,17 @@

Loading

from typing import Callable, Dict, List, Optional, Union, Sequence
from deeplake.experimental.convert_to_libdeeplake import dataset_to_libdeeplake  # type: ignore
from deeplake.experimental.util import (
from typing import Callable, Dict, List, Optional, Union
from deeplake.enterprise.convert_to_libdeeplake import dataset_to_libdeeplake  # type: ignore
from deeplake.enterprise.util import (
    create_fetching_schedule,
    find_primary_tensor,
    raise_indra_installation_error,
    verify_base_storage,
)
from deeplake.experimental.util import collate_fn as default_collate  # type: ignore
from deeplake.experimental.libdeeplake_query import query, sample_by
from deeplake.enterprise.libdeeplake_query import query, sample_by
from deeplake.integrations.pytorch.common import (
    PytorchTransformFunction,
    check_tensors,
    remove_intersections,
    get_collate_fn,
    validate_decode_method,
)
from deeplake.util.bugout_reporter import deeplake_reporter
from deeplake.util.dataset import map_tensor_keys

@@ -58,7 +58,7 @@

Loading

        _return_index=None,
        _primary_tensor_name=None,
        _buffer_size=None,
        _tobytes=None,
        _decode_method=None,
    ):
        import_indra_loader()
        self.dataset = dataset

@@ -76,7 +76,7 @@

Loading

        self._return_index = _return_index
        self._primary_tensor_name = _primary_tensor_name or find_primary_tensor(dataset)
        self._buffer_size = _buffer_size
        self._tobytes = _tobytes
        self._decode_method = _decode_method

    def batch(self, batch_size: int, drop_last: bool = False):
        """Returns a batched :class:`DeepLakeDataLoader` object.

@@ -176,14 +176,12 @@

Loading


        Examples:
            >>> import deeplake
            >>> from deeplake.experimental import dataloader
            >>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
            >>> query_ds_train = dataloader(ds_train).query("select * where labels != 5")
            >>> query_ds_train = ds_train.dataloader().query("select * where labels != 5")

            >>> import deeplake
            >>> from deeplake.experimental import query
            >>> ds_train = deeplake.load('hub://activeloop/coco-train')
            >>> query_ds_train = dataloader(ds_train).query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
            >>> query_ds_train = ds_train.dataloader().query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
        """
        all_vars = self.__dict__.copy()
        all_vars["dataset"] = query(self.dataset, query_string)

@@ -212,20 +210,21 @@

Loading

            Sample the dataloader with ``labels == 5`` twice more than ``labels == 6``

            >>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
            >>> sampled_ds = dataloader(ds).sample_by("max_weight(labels == 5: 10, labels == 6: 5)")
            >>> sampled_ds = ds.dataloader().sample_by("max_weight(labels == 5: 10, labels == 6: 5)")

            Sample the dataloader treating `labels` tensor as weights.

            >>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
            >>> sampled_ds = dataloader(ds).sample_by("labels")
            >>> sampled_ds = ds.dataloader().sample_by("labels")

            Sample the dataloader with the given weights;

            >>> ds_train = deeplake.load('hub://activeloop/coco-train')
            >>> weights = list()
            >>> for i in range(0, len(ds_train)):
            >>>     weights.append(i % 5)
            >>> sampled_ds = dataloader(ds).sample_by(weights, replace=False)
            ...     weights.append(i % 5)
            ...
            >>> sampled_ds = ds.dataloader().sample_by(weights, replace=False)

        """
        all_vars = self.__dict__.copy()

@@ -244,7 +243,7 @@

Loading

        prefetch_factor: int = 2,
        distributed: bool = False,
        return_index: bool = True,
        tobytes: Union[bool, Sequence[str]] = False,
        decode_method: Optional[Dict[str, str]] = None,
    ):
        """Returns a :class:`DeepLakeDataLoader` object.


@@ -252,27 +251,35 @@

Loading

        Args:
            num_workers (int): Number of workers to use for transforming and processing the data. Defaults to 0.
            collate_fn (Callable, Optional): merges a list of samples to form a mini-batch of Tensor(s).
            tensors (List[str], Optional): List of tensors to load. If None, all tensors are loaded. Defaults to None.
            num_threads (int, Optional): Number of threads to use for fetching and decompressing the data. If None, the number of threads is automatically determined. Defaults to None.
            tensors (List[str], Optional): List of tensors to load. If None, all tensors are loaded. Defaults to ``None``.
            num_threads (int, Optional): Number of threads to use for fetching and decompressing the data. If ``None``, the number of threads is automatically determined. Defaults to ``None``.
            prefetch_factor (int): Number of batches to transform and collate in advance per worker. Defaults to 2.
            distributed (bool): Used for DDP training. Distributes different sections of the dataset to different ranks. Defaults to False.
            return_index (bool): Used to identify where loader needs to return sample index or not. Defaults to True.
            tobytes (bool, Sequence[str]): If ``True``, samples will not be decompressed and their raw bytes will be returned instead of numpy arrays. Can also be a list of tensors, in which case those tensors alone will not be decompressed.
            distributed (bool): Used for DDP training. Distributes different sections of the dataset to different ranks. Defaults to ``False``.
            return_index (bool): Used to idnetify where loader needs to retur sample index or not. Defaults to ``True``.
            decode_method (Dict[str, str], Optional): A dictionary of decode methods for each tensor. Defaults to ``None``.

                - Supported decode methods are:

                    :'numpy': Default behaviour. Returns samples as numpy arrays.
                    :'tobytes': Returns raw bytes of the samples.
                    :'pil': Returns samples as PIL images. Especially useful when transformation use torchvision transforms, that
                            require PIL images as input. Only supported for tensors with sample_compression='jpeg' or 'png'.

        Returns:
            DeepLakeDataLoader: A :class:`DeepLakeDataLoader` object.

        Raises:
            ValueError: If .to_pytorch() or .to_numpy() has already been called.
            ValueError: If .pytorch() or .numpy() has already been called.
        """
        if self._mode is not None:
            if self._mode == "numpy":
                raise ValueError("Can't call .to_pytorch after .to_numpy()")
            raise ValueError("already called .to_pytorch()")
                raise ValueError("Can't call .pytorch after .numpy()")
            raise ValueError("already called .pytorch()")
        all_vars = self.__dict__.copy()
        all_vars["_num_workers"] = num_workers
        all_vars["_collate"] = collate_fn
        handle_tensors_and_tobytes(tensors, tobytes, self.dataset, all_vars)
        validate_tensors(tensors, self.dataset, all_vars)
        all_vars["_decode_method"] = decode_method
        all_vars["_num_threads"] = num_threads
        all_vars["_prefetch_factor"] = prefetch_factor
        all_vars["_distributed"] = distributed

@@ -287,7 +294,7 @@

Loading

        tensors: Optional[List[str]] = None,
        num_threads: Optional[int] = None,
        prefetch_factor: int = 2,
        tobytes: Union[bool, Sequence[str]] = False,
        decode_method: Optional[Dict[str, str]] = None,
    ):
        """Returns a :class:`DeepLakeDataLoader` object.


@@ -296,21 +303,28 @@

Loading

            tensors (List[str], Optional): List of tensors to load. If None, all tensors are loaded. Defaults to None.
            num_threads (int, Optional): Number of threads to use for fetching and decompressing the data. If None, the number of threads is automatically determined. Defaults to None.
            prefetch_factor (int): Number of batches to transform and collate in advance per worker. Defaults to 2.
            tobytes (bool, Sequence[str]): If ``True``, samples will not be decompressed and their raw bytes will be returned instead of numpy arrays. Can also be a list of tensors, in which case those tensors alone will not be decompressed.
            decode_method (Dict[str, str], Optional): A dictionary of decode methods for each tensor. Defaults to None.

                - Supported decode methods are:-

                    :'numpy': Default behaviour. Returns samples as numpy arrays.
                    :'tobytes': Returns raw bytes of the samples.
                    :'pil': Returns samples as PIL images. Especially useful when transformation use torchvision transforms, that require PIL images as input. Only supported for tensors with sample_compression='jpeg' or 'png'.

        Returns:
            DeepLakeDataLoader: A :class:`DeepLakeDataLoader` object.

        Raises:
            ValueError: If .to_pytorch() or .to_numpy() has already been called.
            ValueError: If .pytorch() or .numpy() has already been called.
        """
        if self._mode is not None:
            if self._mode == "pytorch":
                raise ValueError("Can't call .to_numpy after .to_pytorch()")
            raise ValueError("already called .to_numpy()")
                raise ValueError("Can't call .numpy after .pytorch()")
            raise ValueError("already called .numpy()")
        all_vars = self.__dict__.copy()
        all_vars["_num_workers"] = num_workers
        handle_tensors_and_tobytes(tensors, tobytes, self.dataset, all_vars)
        validate_tensors(tensors, self.dataset, all_vars)
        all_vars["_decode_method"] = decode_method
        all_vars["_tensors"] = self._tensors or tensors
        all_vars["_num_threads"] = num_threads
        all_vars["_prefetch_factor"] = prefetch_factor

@@ -318,62 +332,35 @@

Loading

        return self.__class__(**all_vars)

    def __iter__(self):
        tensors = self._tensors or map_tensor_keys(self.dataset, None)

        # uncompressed tensors will be uncompressed in the workers on python side
        compressed_tensors = check_tensors(self.dataset, tensors)
        dataset = dataset_to_libdeeplake(self.dataset)
        batch_size = self._batch_size or 1
        drop_last = self._drop_last or False
        return_index = self._return_index
        if return_index is None:
            return_index = True

        shuffle = self._shuffle
        if shuffle is None:
            shuffle = False

        transform_fn = self._transform

        num_workers = self._num_workers or 0
        if self._collate is None and self._mode == "pytorch":
            collate_fn = default_collate
        else:
            collate_fn = self._collate
        num_threads = self._num_threads
        prefetch_factor = self._prefetch_factor
        distributed = self._distributed or False

        # only upcast for pytorch, this handles unsupported dtypes
        upcast = self._mode == "pytorch"
        collate_fn = get_collate_fn(self._collate, self._mode)
        upcast = self._mode == "pytorch"  # upcast to handle unsupported dtypes

        primary_tensor_name = self._primary_tensor_name
        buffer_size = self._buffer_size
        if self._tobytes is True:
            raw_tensors = tensors
        elif self._tobytes is False:
            raw_tensors = []
        else:
            raw_tensors = self._tobytes

        compressed_tensors, raw_tensors = remove_intersections(
            compressed_tensors, raw_tensors
        tensors = self._tensors or map_tensor_keys(self.dataset, None)
        dataset = dataset_to_libdeeplake(self.dataset)

        jpeg_png_compressed_tensors = check_tensors(self.dataset, tensors)
        raw_tensors, compressed_tensors = validate_decode_method(
            self._decode_method, tensors, jpeg_png_compressed_tensors
        )
        raw_tensors.extend(compressed_tensors)
        return iter(
            INDRA_LOADER(
                dataset,
                batch_size=batch_size,
                num_threads=num_threads,
                shuffle=shuffle,
                num_workers=num_workers,
                batch_size=self._batch_size,
                num_threads=self._num_threads,
                shuffle=self._shuffle,
                num_workers=self._num_workers,
                collate_fn=collate_fn,
                transform_fn=transform_fn,
                distributed=distributed,
                prefetch_factor=prefetch_factor,
                transform_fn=self._transform,
                distributed=self._distributed,
                prefetch_factor=self._prefetch_factor,
                tensors=tensors,
                drop_last=drop_last,
                drop_last=self._drop_last,
                upcast=upcast,
                return_index=return_index,
                return_index=self._return_index,
                primary_tensor=primary_tensor_name,
                buffer_size=buffer_size,
                raw_tensors=raw_tensors,

@@ -383,14 +370,14 @@

Loading



def dataloader(dataset) -> DeepLakeDataLoader:
    """Returns a :class:`~deeplake.experimental.dataloader.DeepLakeDataLoader` object which can be transformed to either pytorch dataloader or numpy.
    """Returns a :class:`~deeplake.enterprise.dataloader.DeepLakeDataLoader` object which can be transformed to either pytorch dataloader or numpy.


    Args:
        dataset: :class:`~deeplake.core.dataset.Dataset` object on which dataloader needs to be built

    Returns:
        DeepLakeDataLoader: A :class:`~deeplake.experimental.dataloader.DeepLakeDataLoader` object.
        DeepLakeDataLoader: A :class:`~deeplake.enterprise.dataloader.DeepLakeDataLoader` object.


    Examples:

@@ -400,7 +387,7 @@

Loading



        >>> import deeplake
        >>> from deeplake.experimental import dataloader
        >>> from deeplake.enterprise import dataloader
        >>>
        >>> ds_train = deeplake.load('hub://activeloop/fashion-mnist-train')
        >>> train_loader = dataloader(ds_train).numpy()

@@ -452,7 +439,7 @@

Loading

    return DeepLakeDataLoader(dataset)


def handle_tensors_and_tobytes(tensors, tobytes, dataset, all_vars):
def validate_tensors(tensors, dataset, all_vars):
    existing_tensors = all_vars["_tensors"]
    if tensors:
        if "index" in tensors:

@@ -465,11 +452,3 @@

Loading

                "Tensors have already been specified by passing a dictionary to .transform() method"
            )
    all_vars["_tensors"] = existing_tensors or tensors
    if isinstance(tobytes, Sequence):
        tobytes = map_tensor_keys(dataset, tobytes)
        if tobytes and all_vars["_tensors"]:
            tensor_set = set(all_vars["_tensors"])
            for tensor in tobytes:
                if tensor not in tensor_set:
                    raise ValueError(f"tobytes tensor {tensor} not found in tensors.")
    all_vars["_tobytes"] = tobytes
imilarity index 96%
ename from deeplake/experimental/libdeeplake_query.py
ename to deeplake/enterprise/libdeeplake_query.py

@@ -1344,7 +1344,7 @@

Loading

            The dictionary will always have 2 keys, "dataset" and "tensors". The values corresponding to these keys are detailed below:

                - If ``id_1`` and ``id_2`` are None, both the keys will have a single list as their value. This list will contain a dictionary describing changes compared to the previous commit.
                - If only ``id_1`` is provided, both keys will have a tuple of 2 lists as their value. The lists will contain dictionaries describing commitwise differences between commits. The 2 lists will range from current state and ``id_1` to most recent common ancestor the commits respectively.
                - If only ``id_1`` is provided, both keys will have a tuple of 2 lists as their value. The lists will contain dictionaries describing commitwise differences between commits. The 2 lists will range from current state and ``id_1`` to most recent common ancestor the commits respectively.
                - If only ``id_2`` is provided, a ValueError will be raised.
                - If both ``id_1`` and ``id_2`` are provided, both keys will have a tuple of 2 lists as their value. The lists will contain dictionaries describing commitwise differences between commits. The 2 lists will range from ``id_1`` and ``id_2`` to most recent common ancestor the commits respectively.


@@ -1464,7 +1464,6 @@

Loading

        self,
        transform: Optional[Callable] = None,
        tensors: Optional[Sequence[str]] = None,
        tobytes: Union[bool, Sequence[str]] = False,
        num_workers: int = 1,
        batch_size: int = 1,
        drop_last: bool = False,

@@ -1477,29 +1476,37 @@

Loading

        return_index: bool = True,
        pad_tensors: bool = False,
        transform_kwargs: Optional[Dict[str, Any]] = None,
        decode_method: Optional[Dict[str, str]] = None,
    ):
        """Converts the dataset into a pytorch Dataloader.

        Args:
            transform (Callable, Optional): Transformation function to be applied to each sample.
            tensors (List, Optional): Optionally provide a list of tensor names in the ordering that your training script expects. For example, if you have a dataset that has "image" and "label" tensors, if `tensors=["image", "label"]`, your training script should expect each batch will be provided as a tuple of (image, label).
            tobytes (bool): If ``True``, samples will not be decompressed and their raw bytes will be returned instead of numpy arrays. Can also be a list of tensors, in which case those tensors alone will not be decompressed.
            tensors (List, Optional): Optionally provide a list of tensor names in the ordering that your training script expects. For example, if you have a dataset that has "image" and "label" tensors, if ``tensors=["image", "label"]``, your training script should expect each batch will be provided as a tuple of (image, label).
            num_workers (int): The number of workers to use for fetching data in parallel.
            batch_size (int): Number of samples per batch to load. Default value is 1.
            drop_last (bool): Set to True to drop the last incomplete batch, if the dataset size is not divisible by the batch size.
                if ``False`` and the size of dataset is not divisible by the batch size, then the last batch will be smaller. Default value is False.
                if ``False`` and the size of dataset is not divisible by the batch size, then the last batch will be smaller. Default value is ``False``.
                Read torch.utils.data.DataLoader docs for more details.
            collate_fn (Callable, Optional): merges a list of samples to form a mini-batch of Tensor(s). Used when using batched loading from a map-style dataset.
                Read torch.utils.data.DataLoader docs for more details.
            pin_memory (bool): If ``True``, the data loader will copy Tensors into CUDA pinned memory before returning them. Default value is False.
            pin_memory (bool): If ``True``, the data loader will copy Tensors into CUDA pinned memory before returning them. Default value is ``False``.
                Read torch.utils.data.DataLoader docs for more details.
            shuffle (bool): If ``True``, the data loader will shuffle the data indices. Default value is False. Details about how Deep Lake shuffles data can be found at https://docs.activeloop.ai/how-hub-works/shuffling-in-ds.pytorch
            shuffle (bool): If ``True``, the data loader will shuffle the data indices. Default value is False. Details about how Deep Lake shuffles data can be found at `Shuffling in ds.pytorch() <https://docs.activeloop.ai/how-it-works/shuffling-in-ds.pytorch>`_
            buffer_size (int): The size of the buffer used to shuffle the data in MBs. Defaults to 2048 MB. Increasing the buffer_size will increase the extent of shuffling.
            use_local_cache (bool): If ``True``, the data loader will use a local cache to store data. The default cache location is ~/.activeloop/cache, but it can be changed by setting the LOCAL_CACHE_PREFIX environment variable. This is useful when the dataset can fit on the machine and we don't want to fetch the data multiple times for each iteration. Default value is False
            use_local_cache (bool): If ``True``, the data loader will use a local cache to store data. The default cache location is ~/.activeloop/cache, but it can be changed by setting the ``LOCAL_CACHE_PREFIX`` environment variable. This is useful when the dataset can fit on the machine and we don't want to fetch the data multiple times for each iteration. Default value is ``False``
            use_progress_bar (bool): If ``True``, tqdm will be wrapped around the returned dataloader. Default value is True.
            return_index (bool): If ``True``, the returned dataloader will have a key "index" that contains the index of the sample(s) in the original dataset. Default value is True.
            pad_tensors (bool): If ``True``, shorter tensors will be padded to the length of the longest tensor. Default value is False.
            transform_kwargs (optional, Dict[str, Any]): Additional kwargs to be passed to `transform`.
            transform_kwargs (optional, Dict[str, Any]): Additional kwargs to be passed to ``transform``.
            decode_method (Dict[str, str], Optional): A dictionary of decode methods for each tensor. Defaults to ``None``.

                - Supported decode methods are:

                    :'numpy': Default behaviour. Returns samples as numpy arrays.
                    :'tobytes': Returns raw bytes of the samples.
                    :'pil': Returns samples as PIL images. Especially useful when transformation use torchvision transforms, that
                            require PIL images as input. Only supported for tensors with ``sample_compression='jpeg'`` or ``'png'``.

        Returns:
            A torch.utils.data.DataLoader object.

@@ -1520,7 +1527,6 @@

Loading

            self,
            transform=transform,
            tensors=tensors,
            tobytes=tobytes,
            num_workers=num_workers,
            batch_size=batch_size,
            drop_last=drop_last,

@@ -1531,6 +1537,7 @@

Loading

            use_local_cache=use_local_cache,
            return_index=return_index,
            pad_tensors=pad_tensors,
            decode_method=decode_method,
        )

        if use_progress_bar:

@@ -1538,6 +1545,74 @@

Loading

1538	1545		dataset_read(self)
1539	1546		return dataloader
1540	1547
	1548	+	def dataloader(self):
	1549	+	"""Returns a :class:`~deeplake.enterprise.DeepLakeDataLoader` object. To use this, install deeplake with ``pip install deeplake[enterprise]``.
	1550	+
	1551	+	Returns:
	1552	+	~deeplake.enterprise.DeepLakeDataLoader: A :class:`deeplake.enterprise.DeepLakeDataLoader` object.
	1553	+
	1554	+	Examples:
	1555	+
	1556	+	Creating a simple dataloader object which returns a batch of numpy arrays
	1557	+
	1558	+	>>> import deeplake
	1559	+	>>> ds_train = deeplake.load('hub://activeloop/fashion-mnist-train')
	1560	+	>>> train_loader = ds_train.dataloader().numpy()
	1561	+	>>> for i, data in enumerate(train_loader):
	1562	+	... # custom logic on data
	1563	+	... pass
	1564	+
	1565	+
	1566	+	Creating dataloader with custom transformation and batch size
	1567	+
	1568	+	>>> import deeplake
	1569	+	>>> import torch
	1570	+	>>> from torchvision import datasets, transforms, models
	1571	+	>>>
	1572	+	>>> ds_train = deeplake.load('hub://activeloop/fashion-mnist-train')
	1573	+	>>> tform = transforms.Compose([
	1574	+	... transforms.ToPILImage(), # Must convert to PIL image for subsequent operations to run
	1575	+	... transforms.RandomRotation(20), # Image augmentation
	1576	+	... transforms.ToTensor(), # Must convert to pytorch tensor for subsequent operations to run
	1577	+	... transforms.Normalize([0.5], [0.5]),
	1578	+	... ])
	1579	+	...
	1580	+	>>> batch_size = 32
	1581	+	>>> # create dataloader by chaining with transform function and batch size and returns batch of pytorch tensors
	1582	+	>>> train_loader = ds_train.dataloader()\\
	1583	+	... .transform({'images': tform, 'labels': None})\\
	1584	+	... .batch(batch_size)\\
	1585	+	... .shuffle()\\
	1586	+	... .pytorch()
	1587	+	...
	1588	+	>>> # loop over the elements
	1589	+	>>> for i, data in enumerate(train_loader):
	1590	+	... # custom logic on data
	1591	+	... pass
	1592	+
	1593	+	Creating dataloader and chaining with query
	1594	+
	1595	+	>>> ds = deeplake.load('hub://activeloop/coco-train')
	1596	+	>>> train_loader = ds_train.dataloader()\\
	1597	+	... .query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")\\
	1598	+	... .pytorch()
	1599	+	...
	1600	+	>>> # loop over the elements
	1601	+	>>> for i, data in enumerate(train_loader):
	1602	+	... # custom logic on data
	1603	+	... pass
	1604	+
	1605	+	Restrictions
	1606	+
	1607	+	The new high performance C++ dataloader is part of our Growth and Enterprise Plan .
	1608	+
	1609	+	- Users of our Community plan can create dataloaders on Activeloop datasets ("hub://activeloop/..." datasets).
	1610	+	- To run queries on your own datasets, `upgrade your organization's plan <https://www.activeloop.ai/pricing/>`_.
	1611	+	"""
	1612	+	from deeplake.enterprise import dataloader
	1613	+
	1614	+	return dataloader(self)
	1615	+
1541	1616		@deeplake_reporter.record_call
1542	1617		def filter(
1543	1618		self,

@@ -1591,7 +1666,7 @@

Loading

        return ret

    def query(self, query_string: str):
        """Returns a sliced :class:`~deeplake.core.dataset.Dataset` with given query results.
        """Returns a sliced :class:`~deeplake.core.dataset.Dataset` with given query results. To use this, install deeplake with ``pip install deeplake[enterprise]``.

        It allows to run SQL like queries on dataset and extract results. See supported keywords and the Tensor Query Language documentation
        :ref:`here <tql>`.

@@ -1616,8 +1691,15 @@

Loading


            >>> ds_train = deeplake.load('hub://activeloop/coco-train')
            >>> query_ds_train = ds_train.query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")

        **Restrictions**

        Querying datasets is part of our Growth and Enterprise Plan .

        - Users of our Community plan can only perform queries on Activeloop datasets ("hub://activeloop/..." datasets).
        - To run queries on your own datasets, `upgrade your organization's plan <https://www.activeloop.ai/pricing/>`_.
        """
        from deeplake.experimental import query
        from deeplake.enterprise import query

        return query(self, query_string)


@@ -1627,14 +1709,13 @@

Loading

        replace: Optional[bool] = True,
        size: Optional[int] = None,
    ):
        """Returns a sliced :class:`~deeplake.core.dataset.Dataset` with given weighted sampler applied
        """Returns a sliced :class:`~deeplake.core.dataset.Dataset` with given weighted sampler applied.
        To use this, install deeplake with ``pip install deeplake[enterprise]``.

        Args:
            weights: (Union[str, list, tuple]): If it's string then tql will be run to calculate the weights based on the expression. list and tuple will be treated as the list of the weights per sample
            replace: Optional[bool] If true the samples can be repeated in the result view.
                (default: ``True``).
            size: Optional[int] The length of the result view.
                (default: ``len(dataset)``)
            weights: (Union[str, list, tuple]): If it's string then tql will be run to calculate the weights based on the expression. list and tuple will be treated as the list of the weights per sample.
            replace: Optional[bool] If true the samples can be repeated in the result view. Defaults to ``True``
            size: Optional[int] The length of the result view. Defaults to length of the dataset.


        Returns:

@@ -1651,19 +1732,26 @@

Loading

            Sample the dataset treating `labels` tensor as weights.

            >>> import deeplake
            >>> from deeplake.experimental import query
            >>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
            >>> sampled_ds = ds.sample_by("labels")
            >>> sampled_ds = ds.sample_by("max_weight(labels == 5: 10, labels == 6: 5"))

            Sample the dataset with the given weights;

            >>> ds = deeplake.load('hub://activeloop/coco-train')
            >>> weights = list()
            >>> for i in range(0, len(ds)):
            >>>     weights.append(i % 5)
            >>> for i in range(len(ds)):
            ...     weights.append(i % 5)
            ...
            >>> sampled_ds = ds.sample_by(weights, replace=False)

        **Restrictions**

        Querying datasets is part of our Growth and Enterprise Plan .

        - Users of our Community plan can only use ``sample_by`` on Activeloop datasets ("hub://activeloop/..." datasets).
        - To use sampling functionality on your own datasets, `upgrade your organization's plan <https://www.activeloop.ai/pricing/>`_.
        """
        from deeplake.experimental import sample_by
        from deeplake.enterprise import sample_by

        return sample_by(self, weights, replace, size)


@@ -24,6 +24,7 @@

Loading

    LocalProvider,
)
from deeplake.core.tiling.deserialize import combine_chunks
from deeplake.integrations.pytorch.common import check_tensors, validate_decode_method
from deeplake.util.exceptions import (
    DatasetUnsupportedPytorch,
    SampleDecompressionError,

@@ -31,6 +32,7 @@

Loading

from deeplake.util.keys import get_chunk_key, get_tensor_meta_key
from deeplake.util.remove_cache import get_base_storage
from deeplake.util.storage import get_pytorch_local_storage
from PIL import Image  # type: ignore


ChunkEngineMap = Dict[str, ChunkEngine]

@@ -261,10 +263,11 @@

Loading

        self,
        dataset,
        tensors: Sequence[str],
        tobytes: Union[bool, Sequence[str]] = False,
        use_local_cache: bool = False,
        return_index: bool = True,
        pad_tensors: bool = False,
        decode_method: Optional[Dict[str, str]] = None,
        tobytes: Union[bool, Sequence[str]] = False,
    ) -> None:
        super().__init__()


@@ -282,15 +285,15 @@

Loading


        self.tensors = tensors
        self.pad_tensors = pad_tensors
        if isinstance(tobytes, bool):
            self.tobytes = {k: tobytes for k in self.tensors}
        else:
            for k in tobytes:
                if k not in tensors:
                    raise Exception(
                        f"Tensor {k} is not present in the list of provided tensors: {tensors}."
                    )
            self.tobytes = {k: k in tobytes for k in tensors}
        self.decode_method = decode_method
        self.return_index = return_index

        jpeg_png_compressed_tensors = check_tensors(self.dataset, tensors)
        raw_tensors, compressed_tensors = validate_decode_method(
            self.decode_method, tensors, jpeg_png_compressed_tensors
        )
        self.raw_tensors = set(raw_tensors)
        self.compressed_tensors = set(compressed_tensors)

        self.chunk_engines: ChunkEngineMap = self._map_chunk_engines(self.tensors)


@@ -300,8 +303,6 @@

Loading

            else None
        )

        self.return_index = return_index

    def read(self, schedule: Schedule) -> Iterator:
        for block in schedule._blocks:
            yield from self.stream(block)

@@ -313,7 +314,8 @@

Loading

            valid_sample_flag = True

            for keyid, (key, engine) in enumerate(self.chunk_engines.items()):
                decompress = not self.tobytes[key]
                decompress = key not in self.raw_tensors
                to_pil = key in self.compressed_tensors
                chunk_class = engine.chunk_class
                try:


@@ -347,7 +349,7 @@

Loading

                        chunks.append(chunk)
                    if len(chunks) == 1:
                        data = engine.read_sample_from_chunk(
                            idx, chunk, decompress=decompress
                            idx, chunk, decompress=decompress, to_pil=to_pil
                        )
                    else:
                        if not decompress:

@@ -355,6 +357,8 @@

Loading

                                "`tobytes=True` is not supported by tiled samples as it can cause recompression."
                            )
                        data = combine_chunks(chunks, idx, engine.tile_encoder)
                        if to_pil:
                            data = Image.fromarray(data)  # type: ignore

                    if data is not None:
                        sample[key] = data

@@ -1, +1,6 @@

Loading

1		-	from deeplake.core.transform.transform import compute, compose
	1	+	from deeplake.core.transform.transform import (
	2	+	compute,
	3	+	compose,
	4	+	ComputeFunction,
	5	+	Pipeline,
	6	+	)

@@ -12,7 +12,6 @@

Loading

def create_dataloader_nesteddataloader(
    dataset,
    tensors,
    tobytes,
    use_local_cache,
    transform,
    num_workers,

@@ -23,6 +22,7 @@

Loading

    drop_last,
    return_index,
    pad_tensors,
    decode_method,
):
    import torch
    import torch.utils.data

@@ -34,7 +34,6 @@

Loading

        SubIterableDataset(
            dataset,
            tensors=tensors,
            tobytes=tobytes,
            use_local_cache=use_local_cache,
            transform=transform,
            batch_size=batch_size,

@@ -42,6 +41,7 @@

Loading

            buffer_size=buffer_size,
            return_index=return_index,
            pad_tensors=pad_tensors,
            decode_method=decode_method,
        ),
        batch_size=batch_size,
        collate_fn=collate_fn,

@@ -101,9 +101,9 @@

Loading

    use_local_cache: bool,
    transform: Optional[Union[Dict, Callable]] = None,
    tensors: Optional[Sequence[str]] = None,
    tobytes: Union[bool, Sequence[str]] = False,
    return_index: bool = True,
    pad_tensors: bool = True,
    decode_method: Optional[Dict[str, str]] = None,
):

    import torch

@@ -126,13 +126,10 @@

Loading

    else:
        transform = PytorchTransformFunction(composite_transform=transform)

    check_tensors(dataset, tensors)

    if shuffle and num_workers > 0:
        return create_dataloader(
            dataset,
            tensors,
            tobytes,
            use_local_cache,
            transform,
            num_workers,

@@ -143,13 +140,13 @@

Loading

            drop_last,
            return_index,
            pad_tensors,
            decode_method,
        )
    else:
        return torch.utils.data.DataLoader(
            TorchDataset(
                dataset,
                tensors=tensors,
                tobytes=tobytes,
                use_local_cache=use_local_cache,
                transform=transform,
                num_workers=num_workers,

@@ -157,6 +154,7 @@

Loading

                buffer_size=buffer_size,
                return_index=return_index,
                pad_tensors=pad_tensors,
                decode_method=decode_method,
            ),
            batch_size=batch_size,
            collate_fn=collate_fn,

@@ -1,4 +1,4 @@

Loading

from deeplake.experimental.util import raise_indra_installation_error, remove_tiled_samples  # type: ignore
from deeplake.enterprise.util import raise_indra_installation_error, remove_tiled_samples  # type: ignore
from deeplake.core.storage import S3Provider

from deeplake.util.dataset import try_flushing  # type: ignore
imilarity index 74%
ename from deeplake/experimental/dataloader.py
ename to deeplake/enterprise/dataloader.py

@@ -65,6 +65,7 @@

Loading

        stream: bool = False,
        decompress: bool = True,
        is_tile: bool = False,
        to_pil: bool = False,
    ):
        if self.is_empty_tensor:
            raise EmptyTensorError(

@@ -129,7 +130,10 @@

Loading

            end_idx=stop,
            step=step,
            reverse=reverse,
            to_pil=to_pil,
        )
        if to_pil:
            return sample

        if squeeze:
            sample = sample.squeeze(0)

@@ -11,7 +11,7 @@

Loading

def raise_indra_installation_error(indra_import_error: Optional[Exception] = None):
    if not indra_import_error:
        raise ImportError(
            "This is an experimental feature that requires libdeeplake package. libdeeplake is available only on linux for python versions 3.6 through 3.10 and on macos for python versions 3.7 through 3.10"
            "This is an enterprise feature that requires libdeeplake package which can be installed using pip install deeplake[enterprise]. libdeeplake is available only on linux for python versions 3.6 through 3.10 and on macos for python versions 3.7 through 3.10"
        )
    raise ImportError(
        "Error while importing C++ backend. One of the dependencies might not be installed."

@@ -1,6 +1,6 @@

Loading

from typing import Optional, Union
from deeplake.experimental.convert_to_libdeeplake import dataset_to_libdeeplake
from deeplake.enterprise.convert_to_libdeeplake import dataset_to_libdeeplake
from deeplake.util.bugout_reporter import deeplake_reporter
from typing import Optional, Union

import numpy as np


@@ -26,7 +26,7 @@

Loading

        Query from dataset all the samples with lables other than ``5``

        >>> import deeplake
        >>> from deeplake.experimental import query
        >>> from deeplake.enterprise import query
        >>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
        >>> query_ds_train = query(ds_train, "select * where labels != 5")

imilarity index 89%
ename from deeplake/experimental/test_pytorch.py
ename to deeplake/enterprise/test_pytorch.py

@@ -1,4 +1,3 @@

Loading

1		-	import io
2	1		from logging import warning
3	2		import deeplake
4	3		from deeplake.util.exceptions import (

@@ -18,7 +17,7 @@

Loading

from typing import Union, Tuple, Sequence, List, Optional, BinaryIO
import numpy as np
from pathlib import Path
from PIL import Image, UnidentifiedImageError  # type: ignore
from PIL import Image  # type: ignore
from io import BytesIO

import mmap

@@ -248,7 +247,8 @@

Loading

    end_idx: Optional[int] = None,
    step: Optional[int] = None,
    reverse: bool = False,
) -> np.ndarray:
    to_pil: bool = False,
) -> Union[np.ndarray, Image.Image]:
    """Decompress some buffer into a numpy array. It is expected that all meta information is
    stored inside `buffer`.


@@ -265,13 +265,14 @@

Loading

        end_idx: (int, Optional): Applicable only for video compressions. Index of last frame (exclusive).
        step: (int, Optional): Applicable only for video compressions. Step size for seeking.
        reverse (bool): Applicable only for video compressions. Reverses output numpy array if set to True.
        to_pil (bool): If True, will return a PIL image instead of a numpy array.

    Raises:
        SampleDecompressionError: If decompression fails.
        ValueError: If dtype and shape are not specified for byte compression.

    Returns:
        np.ndarray: Array from the decompressed buffer.
        Union[np.ndarray, Image.Image]: Decompressed array or PIL image.
    """
    compr_type = get_compression_type(compression)
    if compr_type == BYTE_COMPRESSION:

@@ -304,6 +305,8 @@

Loading

        if not isinstance(buffer, str):
            buffer = BytesIO(buffer)  # type: ignore
        img = Image.open(buffer)  # type: ignore
        if to_pil:
            return img
        arr = np.array(img)
        if shape is not None:
            arr = arr.reshape(shape)

@@ -81,6 +81,7 @@

Loading

        self._uncompressed_bytes = None

        self._array = None
        self._pil = None
        self._typestr = None
        self._shape = shape or None
        self._dtype = dtype or None

@@ -300,8 +301,8 @@

Loading

            self._compressed_bytes[compression] = compressed_bytes
        return compressed_bytes

    def _decompress(self):
        if self._array is not None:
    def _decompress(self, to_pil: bool = False):
        if not to_pil and self._array is not None:
            if self._uncompressed_bytes is None:
                self._uncompressed_bytes = self._array.tobytes()
            return

@@ -323,12 +324,24 @@

Loading

            else:
                compressed = self.buffer

            self._array = decompress_array(
                compressed, compression=compression, shape=self.shape, dtype=self.dtype
            )
            self._uncompressed_bytes = self._array.tobytes()
            self._typestr = self._array.__array_interface__["typestr"]
            self._dtype = np.dtype(self._typestr).name
            if to_pil:
                self._pil = decompress_array(
                    compressed,
                    compression=compression,
                    shape=self.shape,
                    dtype=self.dtype,
                    to_pil=True,
                )  # type: ignore
            else:
                self._array = decompress_array(
                    compressed,
                    compression=compression,
                    shape=self.shape,
                    dtype=self.dtype,
                )
                self._uncompressed_bytes = self._array.tobytes()
                self._typestr = self._array.__array_interface__["typestr"]
                self._dtype = np.dtype(self._typestr).name

    def uncompressed_bytes(self) -> Optional[bytes]:
        """Returns uncompressed bytes."""

@@ -352,6 +365,23 @@

Loading

        self._decompress()
        return self._array  # type: ignore

    @property
    def pil(self) -> Image.Image:  # type: ignore
        """Return PIL image corresponding to the sample. Decompresses the sample if necessary.

        Example:

            >>> sample = deeplake.read("./images/dog.jpg")
            >>> pil = sample.pil
            >>> pil.size
            (480, 323)
        """
        pil = self._pil
        if pil is not None:
            return pil
        self._decompress(to_pil=True)
        return self._pil

    def __str__(self):
        if self.is_lazy:
            return f"Sample(is_lazy=True, path={self.path})"

@@ -1,4 +1,5 @@

Loading

1	1		from typing import Callable, Dict, List, Optional
	2	+	import warnings
2	3		from deeplake.util.exceptions import EmptyTensorError
3	4		from deeplake.util.iterable_ordered_dict import IterableOrderedDict
4	5		from deeplake.core.polygon import Polygons

@@ -82,9 +83,42 @@

Loading

    return compressed_tensors


def remove_intersections(compressed_tensors: List[str], raw_tensors: List[str]):
    compressed_tensors = [
        tensor for tensor in compressed_tensors if tensor not in raw_tensors
    ]
    raw_tensors.extend(compressed_tensors)
    return compressed_tensors, raw_tensors
def validate_decode_method(decode_method, all_tensor_keys, jpeg_png_compressed_tensors):
    raw_tensors = []
    compressed_tensors = []
    if decode_method is None:
        if len(jpeg_png_compressed_tensors) > 0:
            warnings.warn(
                f"Decode method for tensors {jpeg_png_compressed_tensors} is defaulting to numpy. Please consider specifying a decode_method in .pytorch() that maximizes the data preprocessing speed based on your transformation."
            )
        return raw_tensors, compressed_tensors

    jpeg_png_compressed_tensors_set = set(jpeg_png_compressed_tensors)
    generic_supported_decode_methods = {"numpy", "tobytes"}
    jpeg_png_supported_decode_methods = {"numpy", "tobytes", "pil"}
    for tensor_name, decode_method in decode_method.items():
        if tensor_name not in all_tensor_keys:
            raise ValueError(
                f"decode_method tensor {tensor_name} not found in tensors."
            )
        if tensor_name in jpeg_png_compressed_tensors_set:
            if decode_method not in jpeg_png_supported_decode_methods:
                raise ValueError(
                    f"decode_method {decode_method} not supported for tensor {tensor_name}. Supported methods for this tensor are {jpeg_png_supported_decode_methods}"
                )
        elif decode_method not in generic_supported_decode_methods:
            raise ValueError(
                f"decode_method {decode_method} not supported for tensor {tensor_name}. Supported methods for this tensor are {generic_supported_decode_methods}"
            )
        if decode_method == "tobytes":
            raw_tensors.append(tensor_name)
        elif decode_method == "pil":
            compressed_tensors.append(tensor_name)

    return raw_tensors, compressed_tensors


def get_collate_fn(collate, mode):
    if collate is None and mode == "pytorch":
        return collate_fn
    return collate

@@ -0,0 +1,3 @@

Loading

	1	+	from deeplake.enterprise.dataloader import dataloader, DeepLakeDataLoader
	2	+	from deeplake.enterprise.libdeeplake_query import query, sample_by
	3	+	from deeplake.enterprise.convert_to_libdeeplake import dataset_to_libdeeplake
0	4		imilarity index 96%
1	5		ename from deeplake/experimental/convert_to_libdeeplake.py
2	6		ename to deeplake/enterprise/convert_to_libdeeplake.py

@@ -64,7 +64,7 @@

Loading

            ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, data_in will be overwritten if it is a Deep Lake dataset, otherwise error will be raised.
                It should have all keys being generated in output already present as tensors. It's initial state should be either:-
                - Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
                - All tensors are populated and have sampe length. In this case new samples are appended to the dataset.
                - All tensors are populated and have same length. In this case new samples are appended to the dataset.
            num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
            scheduler (str): The scheduler to be used to compute the transformation. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
                Defaults to 'threaded'.

@@ -120,31 +120,51 @@

Loading

        pad_data_in: bool = False,
        **kwargs,
    ):
        """Evaluates the pipeline on data_in to produce an output dataset ds_out.
        """Evaluates the pipeline on ``data_in`` to produce an output dataset ``ds_out``.

        Args:
            data_in: Input passed to the transform to generate output dataset. Should support \__getitem__ and \__len__. Can be a Deep Lake dataset.
            ds_out (Dataset, optional): The dataset object to which the transform will get written. If this is not provided, data_in will be overwritten if it is a Deep Lake dataset, otherwise error will be raised.
                It should have all keys being generated in output already present as tensors. It's initial state should be either:-
                - Empty i.e. all tensors have no samples. In this case all samples are added to the dataset.
                - All tensors are populated and have sampe length. In this case new samples are appended to the dataset.
            ds_out (Dataset, optional): - The dataset object to which the transform will get written. If this is not provided, ``data_in`` will be overwritten if it is a Deep Lake dataset, otherwise error will be raised.
                - It should have all keys being generated in output already present as tensors. It's initial state should be either:
                - **Empty**, i.e., all tensors have no samples. In this case all samples are added to the dataset.
                - **All tensors are populated and have same length.** In this case new samples are appended to the dataset.
            num_workers (int): The number of workers to use for performing the transform. Defaults to 0. When set to 0, it will always use serial processing, irrespective of the scheduler.
            scheduler (str): The scheduler to be used to compute the transformation. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
                Defaults to 'threaded'.
            progressbar (bool): Displays a progress bar if True (default).
            skip_ok (bool): If True, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
                This is especially useful for inplace transformations in which certain tensors are not modified. Defaults to False.
            check_lengths (bool): If True, checks whether ds_out has tensors of same lengths initially.
            pad_data_in (bool): NOTE: This is only applicable if data_in is a Deep Lake dataset. If True, pads tensors of data_in to match the length of the largest tensor in data_in.
                Defaults to False.
            progressbar (bool): Displays a progress bar if ``True`` (default).
            skip_ok (bool): If ``True``, skips the check for output tensors generated. This allows the user to skip certain tensors in the function definition.
                This is especially useful for inplace transformations in which certain tensors are not modified. Defaults to ``False``.
            check_lengths (bool): If ``True``, checks whether ``ds_out`` has tensors of same lengths initially.
            pad_data_in (bool): If ``True``, pads tensors of ``data_in`` to match the length of the largest tensor in ``data_in``.
                Defaults to ``False``.
            **kwargs: Additional arguments.

        Raises:
            InvalidInputDataError: If data_in passed to transform is invalid. It should support \__getitem__ and \__len__ operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as data_in will also raise this.
            InvalidOutputDatasetError: If all the tensors of ds_out passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ds_out will also raise this.
            InvalidInputDataError: If ``data_in`` passed to transform is invalid. It should support \__getitem__ and \__len__ operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``data_in`` will also raise this.
            InvalidOutputDatasetError: If all the tensors of ``ds_out`` passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``ds_out`` will also raise this.
            TensorMismatchError: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
            UnsupportedSchedulerError: If the scheduler passed is not recognized. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.
            TransformError: All other exceptions raised if there are problems while running the pipeline.

        Example::

            @deeplake.compute
            def my_fn(sample_in: Any, samples_out, my_arg0, my_arg1=0):
                samples_out.my_tensor.append(my_arg0 * my_arg1)

            # This transform can be used using the eval method in one of these 2 ways:-

            # Directly evaluating the method
            # here arg0 and arg1 correspond to the 3rd and 4th argument in my_fn
            my_fn(arg0, arg1).eval(data_in, ds_out, scheduler="threaded", num_workers=5)

            # As a part of a Transform pipeline containing other functions
            pipeline = deeplake.compose([my_fn(a, b), another_function(x=2)])
            pipeline.eval(data_in, ds_out, scheduler="processed", num_workers=2)

        Note:
            ``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.

        """
        num_workers, scheduler = sanitize_workers_scheduler(num_workers, scheduler)
        overwrite = ds_out is None

@@ -449,20 +469,27 @@

Loading


        - Supported values include: 'serial', 'threaded', 'processed' and 'ray'. Defaults to 'threaded'.

    - ``progressbar (bool)``: Displays a progress bar if True (default).
    - ``progressbar (bool)``: Displays a progress bar if ``True`` (default).

    - ``skip_ok (bool)``: If True, skips the check for output tensors generated.
    - ``skip_ok (bool)``: If ``True``, skips the check for output tensors generated.

        - This allows the user to skip certain tensors in the function definition.
        - This is especially useful for inplace transformations in which certain tensors are not modified. Defaults to ``False``.

    - ``check_lengths (bool)``: If ``True``, checks whether ``ds_out`` has tensors of same lengths initially.

    - ``pad_data_in (bool)``: If ``True``, pads tensors of ``data_in`` to match the length of the largest tensor in ``data_in``. Defaults to ``False``.

    Note:
        ``pad_data_in`` is only applicable if ``data_in`` is a Deep Lake dataset.

    It raises the following errors:

    - ``InvalidInputDataError``: If data_in passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as data_in will also raise this.
    - ``InvalidInputDataError``: If ``data_in`` passed to transform is invalid. It should support ``__getitem__`` and ``__len__`` operations. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``data_in`` will also raise this.

    - ``InvalidOutputDatasetError``: If all the tensors of ds_out passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ds_out will also raise this.
    - ``InvalidOutputDatasetError``: If all the tensors of ``ds_out`` passed to transform don't have the same length. Using scheduler other than "threaded" with deeplake dataset having base storage as memory as ``ds_out`` will also raise this.

    - ``TensorMismatchError``: If one or more of the outputs generated during transform contain different tensors than the ones present in 'ds_out' provided to transform.
    - ``TensorMismatchError``: If one or more of the outputs generated during transform contain different tensors than the ones present in ``ds_out`` provided to transform.

    - ``UnsupportedSchedulerError``: If the scheduler passed is not recognized. Supported values include: 'serial', 'threaded', 'processed' and 'ray'.


1490	1492		local_sample_index = enc.translate_index_relative_to_chunks(
1491	1493		global_sample_index
1492	1494		)
	1495	+	if to_pil:
	1496	+	assert isinstance(chunk, SampleCompressedChunk)
	1497	+	return chunk.read_sample(
	1498	+	local_sample_index,
	1499	+	cast=cast,
	1500	+	copy=copy,
	1501	+	decompress=decompress,
	1502	+	to_pil=True,
	1503	+	)
	1504	+
1493	1505		return chunk.read_sample(
1494	1506		local_sample_index, cast=cast, copy=copy, decompress=decompress
1495	1507		)

402	406		def __init__(
403	407		self,
404	408		dataset,
	409	+	tensors: Sequence[str],
405	410		use_local_cache: bool = False,
406		-	tensors: Optional[Sequence[str]] = None,
407		-	tobytes: Union[bool, Sequence[str]] = False,
408	411		transform: Optional[PytorchTransformFunction] = PytorchTransformFunction(),
409	412		num_workers: int = 1,
410	413		shuffle: bool = False,
411	414		buffer_size: int = 0,
412	415		return_index: bool = True,
413	416		pad_tensors: bool = False,
	417	+	decode_method: Optional[Dict[str, str]] = None,
414	418		) -> None:
415	419		super().__init__()
416	420
417	421		self.dataset = dataset
418	422		self.transform = transform
419	423		self.tensors = tensors
420		-	self.tobytes = tobytes
	424	+	self.shuffle: bool = shuffle
	425	+	self.buffer_size: int = buffer_size * MB
	426	+	self.return_index: bool = return_index
421	427		self.pad_tensors = pad_tensors
	428	+	self.decode_method = decode_method
422	429
423	430		self.use_local_cache = use_local_cache
424	431		self.scheduler = use_scheduler(num_workers, shuffle)

639	640		assert s == sum(list(range(254)))
640	641
641	642
	643	+	def identity(x):
	644	+	return x
	645	+
	646	+
642	647		@requires_torch
643	648		@enabled_non_gdrive_datasets
644	649		@pytest.mark.parametrize("compression", [None, "jpeg"])
645		-	def test_pytorch_tobytes(ds, compressed_image_paths, compression):
	650	+	def test_pytorch_decode(ds, compressed_image_paths, compression):
646	651		with ds:
647	652		ds.create_tensor("image", sample_compression=compression)
648	653		ds.image.extend(

654	659		ds.pytorch()
655	660		return
656	661
657		-	for i, batch in enumerate(ds.pytorch(tobytes=["image"])):
	662	+	for i, batch in enumerate(ds.pytorch(decode_method={"image": "tobytes"})):
658	663		image = batch["image"][0]
659	664		assert isinstance(image, bytes)
660	665		if i < 5 and not compression:

666	671		with open(compressed_image_paths["jpeg"][0], "rb") as f:
667	672		assert f.read() == image
668	673
	674	+	if compression:
	675	+	ptds = ds.pytorch(decode_method={"image": "pil"}, collate_fn=identity)
	676	+	for i, batch in enumerate(ptds):
	677	+	image = batch[0]["image"]
	678	+	assert isinstance(image, Image.Image)
	679	+	if i < 5:
	680	+	np.testing.assert_array_equal(
	681	+	np.array(image), i * np.ones((10, 10, 3), dtype=np.uint8)
	682	+	)
	683	+	elif i >= 5:
	684	+	with Image.open(compressed_image_paths["jpeg"][0]) as f:
	685	+	np.testing.assert_array_equal(np.array(f), np.array(image))
	686	+
669	687
670	688		@requires_torch
671	689		def test_rename(local_ds):

94	94		from deeplake.core.sample import Sample
95	95		from itertools import chain, repeat
96	96		from collections.abc import Iterable
	97	+	from PIL import Image # type: ignore
97	98
98	99
99	100		class ChunkEngine:

1481	1482		cast: bool = True,
1482	1483		copy: bool = False,
1483	1484		decompress: bool = True,
1484		-	) -> np.ndarray:
	1485	+	to_pil: bool = False,
	1486	+	) -> Union[np.ndarray, Image.Image]:
1485	1487		enc = self.chunk_id_encoder
1486	1488		if self.is_fixed_shape and self.sample_compression is None:
1487	1489		num_samples_per_chunk = self.num_samples_per_chunk

18	18		from deeplake.util.link import get_path_creds_key, save_link_creds
19	19		from deeplake.util.video import normalize_index
20	20		import numpy as np
21		-	from typing import Optional, Dict, Any, Tuple
	21	+	from typing import Optional, Dict, Any, Tuple, Union
	22	+	from PIL import Image # type: ignore
22	23
23	24
24	25		def retry_refresh_managed_creds(fn):

251	253		return self.get_empty_sample()
252	254		sample_creds_encoded = creds_encoder.get_encoded_creds_key(global_sample_index)
253	255		sample_creds_key = self.link_creds.get_creds_key(sample_creds_encoded)
254		-	return read_linked_sample(sample_path, sample_creds_key, self.link_creds, False)
	256	+	read_sample = read_linked_sample(
	257	+	sample_path, sample_creds_key, self.link_creds, False
	258	+	)
	259	+	if to_pil:
	260	+	return read_sample.pil
	261	+	return read_sample.array
255	262
256	263		def check_link_ready(self):
257	264		missing_keys = self.link_creds.missing_keys

64	65		def copy_tensor(x):
65	66		if isinstance(x, Sample):
66	67		x = x.array
	68	+	if isinstance(x, Image.Image):
	69	+	return x
	70	+
67	71		try:
68	72		copy = cast_type(x)
69	73		except AttributeError:

432	439		streaming = SampleStreaming(
433	440		dataset,
434	441		tensors=self.tensors, # type: ignore
435		-	tobytes=self.tobytes,
436	442		use_local_cache=use_local_cache,
437	443		pad_tensors=self.pad_tensors,
	444	+	decode_method=self.decode_method,
438	445		)
439	446
440	447		self.schedules: List[Schedule] = self.scheduler.schedule(
441	448		streaming.list_blocks()
442	449		)
443	450
444		-	self.shuffle: bool = shuffle
445		-	self.buffer_size: int = buffer_size * MB
446		-	self.return_index: bool = return_index
447		-
448	451		def __iter__(self):
449	452		worker_info = torch.utils.data.get_worker_info()
450	453		schedule: Schedule = self.schedules[0]

455	458		streaming = SampleStreaming(
456	459		self.dataset,
457	460		tensors=self.tensors,
458		-	tobytes=self.tobytes,
459	461		use_local_cache=self.use_local_cache,
460	462		return_index=self.return_index,
461	463		pad_tensors=self.pad_tensors,
	464	+	decode_method=self.decode_method,
462	465		)
463	466
464	467		if self.shuffle:

13	13		from deeplake.constants import KB
14	14
15	15		from deeplake.tests.dataset_fixtures import enabled_non_gdrive_datasets
	16	+	from PIL import Image # type: ignore
16	17
17	18		try:
18	19		from torch.utils.data._utils.collate import default_collate

25	25		assert arr.shape == (900, 900, 3)
26	26		assert arr.dtype == np.uint32
27	27
	28	+	pil_img = flower.pil
	29	+	assert pil_img.size == (464, 513)
	30	+
28	31
29	32		# TODO: test creating Sample with np.ndarray

1	1		import pickle
2		-	from deeplake.experimental import dataloader
3		-
4	2		import deeplake
5	3		import numpy as np
6	4		import pytest

102	101		)
103	102
104	103		sub_ds3 = ds[:5]
105		-	sub_dl3 = dataloader(sub_ds3).pytorch(num_workers=0)
	104	+	sub_dl3 = sub_ds3.dataloader().pytorch(num_workers=0)
106	105
107	106		for _ in range(2):
108	107		for i, batch in enumerate(sub_dl3):

127	126
128	127		if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
129	128		with pytest.raises(ValueError):
130		-	dl = dataloader(ds)
	129	+	dl = ds.dataloader()
131	130		return
132	131
133	132		dl = (
134		-	dataloader(ds)
	133	+	ds.dataloader()
135	134		.batch(1)
136	135		.transform(to_tuple, t1="image", t2="image2")
137	136		.pytorch(num_workers=2)

161	160
162	161		if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
163	162		with pytest.raises(ValueError):
164		-	dl = dataloader(ds)
	163	+	dl = ds.dataloader()
165	164		return
166	165
167		-	dl = dataloader(ds).transform({"image": double, "image2": None}).pytorch()
	166	+	dl = ds.dataloader().transform({"image": double, "image2": None}).pytorch()
168	167
169	168		assert len(dl.dataset) == 16
170	169

209	208
210	209		if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
211	210		with pytest.raises(ValueError):
212		-	dl = dataloader(ds)
	211	+	dl = ds.dataloader()
213	212		return
214	213
215		-	dl = dataloader(ds).pytorch(num_workers=0)
	214	+	dl = ds.dataloader().pytorch(num_workers=0)
216	215
217	216		for _ in range(2):
218	217		for batch in dl:

79	78		)
80	79
81	80		sub_ds = ds[5:]
82		-	sub_dl = dataloader(sub_ds).pytorch(num_workers=0)
	81	+	sub_dl = sub_ds.dataloader().pytorch(num_workers=0)
83	82
84	83		for i, batch in enumerate(sub_dl):
85	84		np.testing.assert_array_equal(

90	89		)
91	90
92	91		sub_ds2 = ds[8:12]
93		-	sub_dl2 = dataloader(sub_ds2).pytorch(num_workers=0)
	92	+	sub_dl2 = sub_ds2.dataloader().pytorch(num_workers=0)
94	93
95	94		for _ in range(2):
96	95		for i, batch in enumerate(sub_dl2):

280	279		base_storage.enable_readonly()
281	280		ds = Dataset(storage=local_ds.storage, read_only=True, verbose=False)
282	281
283		-	ptds = dataloader(ds).pytorch(num_workers=2)
	282	+	ptds = ds.dataloader().pytorch(num_workers=2)
284	283		# no need to check input, only care that readonly works
285	284		for _ in ptds:
286	285		pass

313	312		local_ds.images.pngs.flowers.append(img2)
314	313
315	314		another_ds = deeplake.dataset(local_ds.path)
316		-	dl = dataloader(another_ds).pytorch(return_index=False)
	315	+	dl = another_ds.dataloader().pytorch(return_index=False)
317	316		for i, (cat, flower) in enumerate(dl):
318	317		assert cat[0].shape == another_ds.images.jpegs.cats[i].numpy().shape
319	318		assert flower[0].shape == another_ds.images.pngs.flowers[i].numpy().shape

326	325		local_ds.create_tensor("strings", htype="text")
327	326		local_ds.strings.extend([f"string{idx}" for idx in range(5)])
328	327
329		-	ptds = dataloader(local_ds).pytorch()
	328	+	ptds = local_ds.dataloader().pytorch()
330	329		for idx, batch in enumerate(ptds):
331	330		np.testing.assert_array_equal(batch["strings"], f"string{idx}")
332	331

365	364		ds.img2.extend(arr_list_2)
366	365		ds.label.extend(label_list)
367	366
368		-	ptds = dataloader(local_ds[index]).pytorch()
	367	+	ptds = local_ds[index].dataloader().pytorch()
369	368		idxs = list(IndexEntry(index).indices(len(local_ds)))
370	369		for idx, batch in enumerate(ptds):
371	370		idx = idxs[idx]

387	386		local_ds.b.append(1)
388	387		local_ds.c.append(2)
389	388
390		-	ptds = dataloader(local_ds).batch(4).pytorch(collate_fn=reorder_collate)
	389	+	ptds = local_ds.dataloader().batch(4).pytorch(collate_fn=reorder_collate)
391	390		if shuffle:
392	391		ptds = ptds.shuffle()
393	392		for batch in ptds:

412	411		local_ds.c.append(2 * np.ones((300, 300)))
413	412
414	413		ptds = (
415		-	dataloader(local_ds)
	414	+	local_ds.dataloader()
416	415		.batch(4)
417	416		.pytorch(
418	417		collate_fn=my_transform_collate,

439	438		@requires_libdeeplake
440	439		@enabled_non_gdrive_datasets
441	440		@pytest.mark.parametrize("compression", [None, "jpeg"])
442		-	def test_pytorch_tobytes(ds, compressed_image_paths, compression):
	441	+	def test_pytorch_decode(ds, compressed_image_paths, compression):
443	442		with ds:
444	443		ds.create_tensor("image", sample_compression=compression)
445	444		ds.image.extend(

448	447		ds.image.extend([deeplake.read(compressed_image_paths["jpeg"][0])] * 5)
449	448		if isinstance(get_base_storage(ds.storage), (MemoryProvider, GCSProvider)):
450	449		with pytest.raises(ValueError):
451		-	dl = dataloader(ds)
	450	+	dl = ds.dataloader()
452	451		return
453	452
454		-	ptds = dataloader(ds).pytorch(tobytes=["image"])
	453	+	ptds = ds.dataloader().pytorch(decode_method={"image": "tobytes"})
455	454
456	455		for i, batch in enumerate(ptds):
457	456		image = batch["image"][0]

Updating our web app

No yaml found.

476	488		ds.rename_tensor("abc", "xyz")
477	489		ds.rename_group("blue", "red")
478	490		ds["red/green"].append([1, 2, 3, 4])
479		-	loader = dataloader(ds).pytorch(return_index=False)
	491	+	loader = ds.dataloader().pytorch(return_index=False)
480	492		for sample in loader:
481	493		assert set(sample.keys()) == {"xyz", "red/green"}
482	494		np.testing.assert_array_equal(np.array(sample["xyz"]), np.array([[1, 2, 3]]))

489	501		@requires_libdeeplake
490	502		def test_expiration_date_casting_to_string():
491	503		ds = deeplake.dataset("hub://activeloop/cifar100-train")[0:10:2]
492		-	loader = dataloader(ds).pytorch(return_index=False)
	504	+	loader = ds.dataloader().pytorch(return_index=False)
493	505		for _ in loader:
494	506		pass
495	507

505	517		ds.xyz.append(i * np.ones((2, 2)))
506	518
507	519		ptds = (
508		-	dataloader(local_ds)
	520	+	local_ds.dataloader()
509	521		.batch(4)
510	522		.pytorch(num_workers=num_workers, return_index=True)
511	523		)

571	583		np.testing.assert_array_equal(2 * batch["index"][i], batch["xyz"][i][0, 0])
572	584
573	585		ptds = (
574		-	dataloader(local_ds)
	586	+	local_ds.dataloader()
575	587		.batch(4)
576	588		.transform({"xyz": double})
577	589		.pytorch(num_workers=num_workers, return_index=True)

622	634		ds.create_tensor("y")
623	635		ds.x.extend(list(range(5)))
624	636		ds.y.extend(list(range(10)))
625		-	ptds = dataloader(ds).pytorch()
	637	+	ptds = ds.dataloader().pytorch()
626	638		for i, batch in enumerate(ptds):
627	639		x, y = np.array(batch["x"][0]), np.array(batch["y"][0])
628	640		np.testing.assert_equal(x, i)

1		-	from typing import Callable, Dict, List, Optional, Union, Sequence
2		-	from deeplake.experimental.convert_to_libdeeplake import dataset_to_libdeeplake # type: ignore
3		-	from deeplake.experimental.util import (
	1	+	from typing import Callable, Dict, List, Optional, Union
	2	+	from deeplake.enterprise.convert_to_libdeeplake import dataset_to_libdeeplake # type: ignore
	3	+	from deeplake.enterprise.util import (
4	4		create_fetching_schedule,
5	5		find_primary_tensor,
6	6		raise_indra_installation_error,
7	7		verify_base_storage,
8	8		)
9		-	from deeplake.experimental.util import collate_fn as default_collate # type: ignore
10		-	from deeplake.experimental.libdeeplake_query import query, sample_by
	9	+	from deeplake.enterprise.libdeeplake_query import query, sample_by
11	10		from deeplake.integrations.pytorch.common import (
12	11		PytorchTransformFunction,
13	12		check_tensors,
14		-	remove_intersections,
	13	+	get_collate_fn,
	14	+	validate_decode_method,
15	15		)
16	16		from deeplake.util.bugout_reporter import deeplake_reporter
17	17		from deeplake.util.dataset import map_tensor_keys

58	58		_return_index=None,
59	59		_primary_tensor_name=None,
60	60		_buffer_size=None,
61		-	_tobytes=None,
	61	+	_decode_method=None,
62	62		):
63	63		import_indra_loader()
64	64		self.dataset = dataset

76	76		self._return_index = _return_index
77	77		self._primary_tensor_name = _primary_tensor_name or find_primary_tensor(dataset)
78	78		self._buffer_size = _buffer_size
79		-	self._tobytes = _tobytes
	79	+	self._decode_method = _decode_method
80	80
81	81		def batch(self, batch_size: int, drop_last: bool = False):
82	82		"""Returns a batched :class:`DeepLakeDataLoader` object.

176	176
177	177		Examples:
178	178		>>> import deeplake
179		-	>>> from deeplake.experimental import dataloader
180	179		>>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
181		-	>>> query_ds_train = dataloader(ds_train).query("select * where labels != 5")
	180	+	>>> query_ds_train = ds_train.dataloader().query("select * where labels != 5")
182	181
183	182		>>> import deeplake
184		-	>>> from deeplake.experimental import query
185	183		>>> ds_train = deeplake.load('hub://activeloop/coco-train')
186		-	>>> query_ds_train = dataloader(ds_train).query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
	184	+	>>> query_ds_train = ds_train.dataloader().query("(select * where contains(categories, 'car') limit 1000) union (select * where contains(categories, 'motorcycle') limit 1000)")
187	185		"""
188	186		all_vars = self.__dict__.copy()
189	187		all_vars["dataset"] = query(self.dataset, query_string)

212	210		Sample the dataloader with ``labels == 5`` twice more than ``labels == 6``
213	211
214	212		>>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
215		-	>>> sampled_ds = dataloader(ds).sample_by("max_weight(labels == 5: 10, labels == 6: 5)")
	213	+	>>> sampled_ds = ds.dataloader().sample_by("max_weight(labels == 5: 10, labels == 6: 5)")
216	214
217	215		Sample the dataloader treating `labels` tensor as weights.
218	216
219	217		>>> ds = deeplake.load('hub://activeloop/fashion-mnist-train')
220		-	>>> sampled_ds = dataloader(ds).sample_by("labels")
	218	+	>>> sampled_ds = ds.dataloader().sample_by("labels")
221	219
222	220		Sample the dataloader with the given weights;
223	221
224	222		>>> ds_train = deeplake.load('hub://activeloop/coco-train')
225	223		>>> weights = list()
226	224		>>> for i in range(0, len(ds_train)):
227		-	>>> weights.append(i % 5)
228		-	>>> sampled_ds = dataloader(ds).sample_by(weights, replace=False)
	225	+	... weights.append(i % 5)
	226	+	...
	227	+	>>> sampled_ds = ds.dataloader().sample_by(weights, replace=False)
229	228
230	229		"""
231	230		all_vars = self.__dict__.copy()