47 lines
2.1 KiB
TypeScript
47 lines
2.1 KiB
TypeScript
// The automatic PDF intake's reader (core-11): the platform's document reader (`DocumentReader`, kit-platform's
|
|
// `@conusai-sdk/ocr` wired to the deployment contract) and the read document's passages
|
|
// (`@conusai-sdk/ocr/passages`), cut to the knowledge service's limit for one indexed item. The one thing this
|
|
// adapter decides is what is permanent: a job the platform failed, or a document whose blocks no metadata
|
|
// value can cite (`passages` throws `RangeError`), is not indexable; anything else is thrown for the queue.
|
|
|
|
import type { DocumentReader } from "@conusai-kit/platform"
|
|
import { KNOWLEDGE_LIMITS } from "@conusai-kit/platform"
|
|
import { OcrError } from "@conusai-sdk/ocr"
|
|
import { passages } from "@conusai-sdk/ocr/passages"
|
|
import type { KnowledgeReader } from "../core/ports.js"
|
|
|
|
export interface SdkKnowledgeReaderOptions {
|
|
/** Poll interval of `wait`, in milliseconds. */
|
|
readonly intervalMs?: number
|
|
/** Give up on one job after this long; the queue's retry submits again under the same key. */
|
|
readonly timeoutMs?: number
|
|
}
|
|
|
|
export function sdkKnowledgeReader(
|
|
reader: Pick<DocumentReader, "submit" | "wait">,
|
|
options: SdkKnowledgeReaderOptions = {},
|
|
): KnowledgeReader {
|
|
return {
|
|
async read(bytes, { filename, idempotencyKey, signal }) {
|
|
try {
|
|
const job = await reader.submit(bytes, { filename, idempotencyKey })
|
|
const document = await reader.wait(job.id, {
|
|
...(options.intervalMs === undefined ? {} : { intervalMs: options.intervalMs }),
|
|
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
|
|
...(signal ? { signal } : {}),
|
|
})
|
|
return passages(document, { maxBytes: KNOWLEDGE_LIMITS.inlineTextBytes }).map((passage) => ({
|
|
page: passage.page,
|
|
index: passage.index,
|
|
blockIds: passage.blockIds,
|
|
text: passage.text,
|
|
}))
|
|
} catch (error) {
|
|
if ((error instanceof OcrError && error.type === "job-failed") || error instanceof RangeError)
|
|
throw Object.assign(error, { permanent: "not_indexable" })
|
|
throw error
|
|
}
|
|
},
|
|
}
|
|
}
|