ある日時以降に更新されたすべてのscrapboxのページを取得するscript

前回取得した日時を使えば、更新分だけをfetchできるので、サーバー負荷を低減し処理の高速化に貢献できる

code:mod.ts

import {

BaseOptions,

Fetch,

listPages,

getPage,

InvalidFollowingIdError,

Result,

TooLongURIError,

} from "../scrapbox-userscript-std/rest.ts";

import {

Page,

ExportedPage,

NotFoundError,

NotLoggedInError,

NotMemberError,

} from "../scrapbox-jp%2Ftypes/rest.ts";

export async function* getAllUpdatedPages(project: string, lastChecked: number, options?: BaseOptions): AsyncGenerator<

Result<Page, NotFoundError | NotLoggedInError | NotMemberError | TooLongURIError>,

void,

unknown

{

const throttle = makeThrottle<Response>(5);

const { fetch: fetch_ = globalThis.fetch, ...rest } = options ?? {};

const fetch: Fetch = (args) => throttle(() => fetch_(args));

let skip = 0;

while (true) {

const result = await listPages(project, { ...options, limit: 1000, skip });

if (!result.ok) {

yield result;

return;

}

const promises = result.value.pages.flatMap(

);

for await (const promise of promises) {

yield await promise;

}

if ((result.value.pages.pop()?.updated ?? 0) <= lastChecked) break;

skip += 1000;

}

}