Source code for django_utils.queryset

import gc
from collections.abc import Callable, Iterator
from typing import Any

from django.db.models import QuerySet




def queryset_iterator(
    queryset: QuerySet[Any],
    chunksize: int = 1000,
    getfunc: Callable[[Any, str], Any] = getattr,
) -> Iterator[Any]:
    """''
    Iterate over a Django Queryset ordered by the primary key

    This method loads a maximum of chunksize (default: 1000) rows in it's
    memory at the same time while django normally would load all rows in it's
    memory. Using the iterator() method only causes it to not preload all the
    classes.

    Note that the implementation of the iterator does not support ordered
    query sets.
    """
    pk = 0

    try:
        """In the case of an empty list, return"""
        last_pk = getfunc(queryset.order_by('-pk')[0], 'pk')
    except IndexError:
        return

    queryset = queryset.order_by('pk')
    while pk < last_pk:
        for row in queryset.filter(pk__gt=pk)[:chunksize]:
            pk = getfunc(row, 'pk')
            yield row
        gc.collect()