Pagination

Example Setup

Table Definition

{
  "TableName": "electro",
  "KeySchema": [
    {
      "AttributeName": "pk",
      "KeyType": "HASH"
    },
    {
      "AttributeName": "sk",
      "KeyType": "RANGE"
    }
  ],
  "AttributeDefinitions": [
    {
      "AttributeName": "pk",
      "AttributeType": "S"
    },
    {
      "AttributeName": "sk",
      "AttributeType": "S"
    },
    {
      "AttributeName": "gsi1pk",
      "AttributeType": "S"
    },
    {
      "AttributeName": "gsi1sk",
      "AttributeType": "S"
    }
  ],
  "GlobalSecondaryIndexes": [
    {
      "IndexName": "gsi1pk-gsi1sk-index",
      "KeySchema": [
        {
          "AttributeName": "gsi1pk",
          "KeyType": "HASH"
        },
        {
          "AttributeName": "gsi1sk",
          "KeyType": "RANGE"
        }
      ],
      "Projection": {
        "ProjectionType": "ALL"
      }
    },
    {
      "IndexName": "gsi2pk-gsi2sk-index",
      "KeySchema": [
        {
          "AttributeName": "gsi2pk",
          "KeyType": "HASH"
        },
        {
          "AttributeName": "gsi2sk",
          "KeyType": "RANGE"
        }
      ],
      "Projection": {
        "ProjectionType": "ALL"
      }
    }
  ],
  "BillingMode": "PAY_PER_REQUEST"
}

Example Entity

import DynamoDB from "aws-sdk/clients/dynamodb";
import { Entity } from "electrodb";

const client = new DynamoDB.DocumentClient();

const table = "electro";

const StoreLocations = new Entity(
  {
    model: {
      service: "MallStoreDirectory",
      entity: "MallStore",
      version: "1",
    },
    attributes: {
      cityId: {
        type: "string",
        required: true,
      },
      mallId: {
        type: "string",
        required: true,
      },
      storeId: {
        type: "string",
        required: true,
      },
      buildingId: {
        type: "string",
        required: true,
      },
      unitId: {
        type: "string",
        required: true,
      },
      category: {
        type: [
          "spite store",
          "food/coffee",
          "food/meal",
          "clothing",
          "electronics",
          "department",
          "misc",
        ],
        required: true,
      },
      leaseEndDate: {
        type: "string",
        required: true,
      },
      rent: {
        type: "string",
        required: true,
        validate: /^(\d+\.\d{2})$/,
      },
      discount: {
        type: "string",
        required: false,
        default: "0.00",
        validate: /^(\d+\.\d{2})$/,
      },
      tenants: {
        type: "set",
        items: "string",
      },
      warnings: {
        type: "number",
        default: 0,
      },
      deposit: {
        type: "number",
      },
      contact: {
        type: "set",
        items: "string",
      },
      rentalAgreement: {
        type: "list",
        items: {
          type: "map",
          properties: {
            type: {
              type: "string",
            },
            detail: {
              type: "string",
            },
          },
        },
      },
      petFee: {
        type: "number",
      },
      fees: {
        type: "number",
      },
      tags: {
        type: "set",
        items: "string",
      },
    },
    indexes: {
      stores: {
        pk: {
          field: "pk",
          composite: ["cityId", "mallId"],
        },
        sk: {
          field: "sk",
          composite: ["buildingId", "storeId"],
        },
      },
      units: {
        index: "gsi1pk-gsi1sk-index",
        pk: {
          field: "gsi1pk",
          composite: ["mallId"],
        },
        sk: {
          field: "gsi1sk",
          composite: ["buildingId", "unitId"],
        },
      },
      leases: {
        index: "gsi2pk-gsi2sk-index",
        pk: {
          field: "gsi2pk",
          composite: ["storeId"],
        },
        sk: {
          field: "gsi2sk",
          composite: ["leaseEndDate"],
        },
      },
    },
  },
  { table, client },
);

(Example code on this page that references the entity StoreLocations uses the following Entity and Table Definition found below)

Cursors

All ElectroDB query and scan operations return a cursor, which is a stringified and copy of DynamoDB’s LastEvaluatedKey with a base64url encoding.

The terminal method go() accepts a cursor when executing a query or scan to continue paginating for more results. Pass the cursor from the previous query to your next query and ElectroDB will continue its pagination where it left off.

To limit the number of items ElectroDB will retrieve, read more about the Query Options pages and limit.

Entities

const results1 = await MallStores.query.leases({ mallId }).go(); // no "cursor" passed to `.go()`

const results2 = await MallStores.query.leases({ mallId })
  .go({ cursor: results1.cursor }); // Paginate by querying with the "cursor" from your first query

{
  "cursor": "...",
  "data": [
    {
      "mall": "3010aa0d-5591-4664-8385-3503ece58b1c",
      "leaseEnd": "2020-01-20",
      "sector": "7d0f5c19-ec1d-4c1e-b613-a4cc07eb4db5",
      "store": "MNO",
      "unit": "B5",
      "id": "e0705325-d735-4fe4-906e-74091a551a04",
      "building": "BuildingE",
      "category": "food/coffee",
      "rent": "0.00"
    },
    {
      "mall": "3010aa0d-5591-4664-8385-3503ece58b1c",
      "leaseEnd": "2020-01-20",
      "sector": "7d0f5c19-ec1d-4c1e-b613-a4cc07eb4db5",
      "store": "ZYX",
      "unit": "B9",
      "id": "f201a1d3-2126-46a2-aec9-758ade8ab2ab",
      "building": "BuildingI",
      "category": "food/coffee",
      "rent": "0.00"
    }
  ]
}

Services

Pagination with services is also possible. Similar to Entity Pagination, calling the .go() method returns the following structure:

type GoResults = {
  cursor: string | null;
  data: {
    [entityName: string]: {
      /** EntityItem */
    }[];
  };
};

Execution Options

Count

The execution option count allows you to specify a specific number of items to be returned. This is often a difficult task with DynamoDB because queries do not always return a consistent number of items. If your query includes filters, and requires pagination, it can be even harder to return a specific number of items reliably. When using count, ElectroDB will paginate your query against DynamoDB until the number of items matches the supplied count, create a custom cursor, and return the items found. This option is recommend for queries with numerous and/or strict attribute filters where end-user or external pagination is necessary.

type GetItemsOptions = {
  mallId: string;
  cursor: string;
  limit?: number;
}

async function getLeases(options: GetItemsOptions) {
  const { mallId, cursor, limit } = options;

  if (limit < 1 || limit >= 200) {
    throw new Error('Limit must be at least 1 and at most 200');
  }

  return MallStores.query.leases({ mallId })
    .go({ cursor, count: limit });
}

Pages

The execution option pages allows you to automatically perform multiple queries against DynamoDB. By default, ElectroDB queries will perform one request against DynamoDB. With pages, you can specify the number of queries you’d like to occur under the hood before returning. Furthermore, if you would like ElectroDB to return all results for a given query (i.e., exhausting pagination for a given query automatically) you can use the option {pages: "all"}. Note that while option is convenient, it may not performant for some workflows.

Async Iteration

The result of .go() on query, scan, and collection operations can be used as an AsyncIterable, allowing you to iterate through pages of results using for await...of. Each iteration yields a single page of results with its own cursor.

Entity Query

for await (const page of MallStores.query.leases({ mallId }).go({ pages: "all" })) {
  console.log(page.data);   // items for this page
  console.log(page.cursor);  // cursor for this page (null on last page)
}

Scan

for await (const page of MallStores.scan.go({ pages: "all" })) {
  console.log(page.data);
  console.log(page.cursor);
}

Collection Query

for await (const page of TaskApp.collections.assignments({ employeeId: "JExotic" }).go({ pages: "all" })) {
  console.log(page.data.employees);
  console.log(page.data.tasks);
  console.log(page.cursor);
}

Early Termination

You can break out of the loop at any time to stop fetching additional pages:

const firstBatch: StoreItem[] = [];

for await (const page of MallStores.query.leases({ mallId }).go({ pages: "all" })) {
  firstBatch.push(...page.data);
  if (firstBatch.length >= 100) {
    break; // stop pagination early
  }
}

Combining with Execution Options

All existing execution options work with async iteration. For example, limit controls the DynamoDB Limit per page, and pages controls how many pages to fetch:

// Each page returns at most 25 items, iterate at most 10 pages
for await (const page of MallStores.query.leases({ mallId }).go({ limit: 25, pages: 10 })) {
  console.log(page.data);
}

Example

Simple pagination example using async iteration:

import { EntityItem } from "electrodb";

import { users } from "./entities";

type UserItem = EntityItem<typeof users>;

async function getTeamMembers(team: string) {
  const members: UserItem[] = [];

  for await (const page of users.query.members({ team }).go({ pages: "all" })) {
    members.push(...page.data);
  }

  return members;
}

Simple pagination example using cursors:

// EntityItem is the type for a returned item
// QueryResponse is the type for the full electrodb response to a query
import { EntityItem, QueryResponse } from "electrodb";

// (your entity)
import { users } from "./entities";

type UserItem = EntityItem<typeof users>;
type UserQueryResponse = QueryResponse<typeof users>;

async function getTeamMembers(team: string) {
  let members: UserItem[] = [];
  let cursor = null;
  do {
    const results: UserQueryResponse = await users.query
      .members({ team })
      .go({ cursor });
    members = [...members, ...results.data];
    cursor = results.cursor;
  } while (cursor !== null);

  return members;
}