ConusAI vendor v0.1.0-beta.114

modules/knowledge/adapters/reader.ts

@@ -0,0 +1,46 @@
+// The automatic PDF intake's reader (core-11): the platform's document reader (`DocumentReader`, kit-platform's
+// `@conusai-sdk/ocr` wired to the deployment contract) and the read document's passages
+// (`@conusai-sdk/ocr/passages`), cut to the knowledge service's limit for one indexed item. The one thing this
+// adapter decides is what is permanent: a job the platform failed, or a document whose blocks no metadata
+// value can cite (`passages` throws `RangeError`), is not indexable; anything else is thrown for the queue.
+
+import type { DocumentReader } from "@conusai-kit/platform"
+import { KNOWLEDGE_LIMITS } from "@conusai-kit/platform"
+import { OcrError } from "@conusai-sdk/ocr"
+import { passages } from "@conusai-sdk/ocr/passages"
+import type { KnowledgeReader } from "../core/ports.js"
+
+export interface SdkKnowledgeReaderOptions {
+  /** Poll interval of `wait`, in milliseconds. */
+  readonly intervalMs?: number
+  /** Give up on one job after this long; the queue's retry submits again under the same key. */
+  readonly timeoutMs?: number
+}
+
+export function sdkKnowledgeReader(
+  reader: Pick<DocumentReader, "submit" | "wait">,
+  options: SdkKnowledgeReaderOptions = {},
+): KnowledgeReader {
+  return {
+    async read(bytes, { filename, idempotencyKey, signal }) {
+      try {
+        const job = await reader.submit(bytes, { filename, idempotencyKey })
+        const document = await reader.wait(job.id, {
+          ...(options.intervalMs === undefined ? {} : { intervalMs: options.intervalMs }),
+          ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
+          ...(signal ? { signal } : {}),
+        })
+        return passages(document, { maxBytes: KNOWLEDGE_LIMITS.inlineTextBytes }).map((passage) => ({
+          page: passage.page,
+          index: passage.index,
+          blockIds: passage.blockIds,
+          text: passage.text,
+        }))
+      } catch (error) {
+        if ((error instanceof OcrError && error.type === "job-failed") || error instanceof RangeError)
+          throw Object.assign(error, { permanent: "not_indexable" })
+        throw error
+      }
+    },
+  }
+}