/// <reference lib="deno.ns" /> import { getOCRs } from "../GyazoのURLリストからOCRテキストを一括取得するscript/mod.ts"; import { IndexParseStream, PageSortStream, PageStringifyStream, Page, PageSource } from "../Scrapbox書籍のformat@0.2.0/mod.ts"; import { getGyazoToken } from "../scrapbox-userscript-std/rest.ts"; import { CsvParseStream } from "jsr:@std/csv@1/parse-stream"; import type { ImportedLightPage } from "../scrapbox-jp%2Ftypes/rest.ts"; import { isAbsolute, toFileUrl, resolve } from "jsr:@std/path@1/posix"; import { Command } from "jsr:@cliffy/command@1.0.0-rc.7"; import { Spinner } from "jsr:@std/cli@1/unstable-spinner"; import { isErr, unwrapErr, unwrapOk } from "npm:option-t@50/plain_result"; const { args: [gyazoListPath, csvPath], options: { token, offset, outfile } } = await new Command() .name("Scrapbox書籍を作るUserScript") .description("GyazoのURLリストとscrapbox書籍用目次CSVからScrapbox書籍のJSONファイルを作る") .version("v0.2.0") .arguments("<gyazoListPath:string> <csvPath:string>") .option("-t, --token <token:string>", "Gyazo Access Token") .option("-o, --outfile <outfile:string>", "出力先ファイルパス", { default: "scrapbox.json" }) .option("--offset <offset:number>", "scrapbox書籍用目次CSVに書かれたページ番号をoffsetだけずらす", { default: 0 }) .parse(Deno.args); const listURL = toFileUrl( isAbsolute(gyazoListPath) ? gyazoListPath: resolve(Deno.cwd(), gyazoListPath) ); const gyazoList = (await (await fetch(listURL)).json()) as string[]; // OCRを取り込む let counter = 0; const errors = [] as number[]; const spinner = new Spinner(); const source: AsyncIterable<PageSource> = (async function*() { spinner.message = "Download OCRs..."; spinner.start(); const header = () => `${counter - errors.length}/${gyazoList.length} got, ${errors.length} failed: `; for await (const result of getOCRs(gyazoList, token!)) { counter++; if (isErr(result)) { const reason = unwrapErr(result); if (!(reason instanceof Error)) throw result; console.error(result); errors.push(counter); spinner.message = `${header()}${reason}`; continue; } const source = unwrapOk(result); yield source; spinner.message = `${header()}${[...source.text.replaceAll("\n","")].slice(0, 10).join("")}`; } spinner.message = "creating the json file..."; })(); const csvURL = URL.canParse(csvPath) ? new URL(csvPath) : toFileUrl(isAbsolute(csvPath) ? csvPath : resolve(Deno.cwd(), csvPath) ); const pages = await Array.fromAsync((await fetch(csvURL)).body! .pipeThrough(new TextDecoderStream()) // 目次を取り込む .pipeThrough(new CsvParseStream()) .pipeThrough(new IndexParseStream()) .pipeThrough(new PageSortStream()) .pipeThrough(new PageStringifyStream(source)) // JSONを作る .pipeThrough(new TransformStream<string[], ImportedLightPage>({ transform(lines, controller) { controller.enqueue({ title: lines[0], lines }); } })) ); await Deno.writeTextFile(outfile, JSON.stringify({ pages })); spinner.stop(); console.log("created the json file.");