ConusAI vendor v0.1.0-beta.114

2026-11-14 09:56:45 +00:00
commit 1ce8d23c35
867 changed files with 122104 additions and 0 deletions
@@ -0,0 +1,72 @@
+// The automatic PDF intake (template lesson core-11): what the worker runs for one uploaded file the
+// files module handed to this module's processor. A project that opted in (`knowledge_auto_index`)
+// gets the PDF indexed into its collection as a document like any other, requested by the
+// administrator who opted in: the job runs for nobody else, so that administrator's admission is what
+// each step leaving the process checks again (`beforeIndex`). Anything else reads nothing.
+
+import { AppError } from "@conusai-kit/core"
+import { getFile } from "@conusai-kit/files"
+import { AUTO_INDEX_CONTENT_TYPES, autoIndex, type DocumentView, documents } from "../contract/index.js"
+import { indexPassages } from "./index-job.js"
+import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
+
+/** What the files module's processor job carries about the upload. */
+export interface AutoIndexUpload {
+  fileId: string
+  contentType: string
+  sha256: string
+}
+
+/**
+ * Indexes one upload of an opted-in project, or answers null and reads nothing: a type it does not read,
+ * a project that is not opted in or has knowledge off, a file that is gone, or one whose digest is not
+ * the upload's. A redelivered job finds the document row and resumes it, so the work is done once.
+ */
+export async function autoIndexUpload(
+  deps: KnowledgeDeps,
+  scope: KnowledgeScope,
+  upload: AutoIndexUpload,
+  options: { signal?: AbortSignal; finalAttempt?: boolean } = {},
+): Promise<DocumentView | null> {
+  if (!(AUTO_INDEX_CONTENT_TYPES as readonly string[]).includes(upload.contentType)) return null
+  const optIn = (await deps.data.forScope(scope).of(autoIndex).list({ limit: 1 })).items[0]
+  if (!optIn) return null
+  if (!(await knowledgeEnabled(deps, scope))) return null
+  const file = await storedFile(deps, scope, upload.fileId)
+  if (!file || file.sha256 !== upload.sha256) return null
+
+  const row = await deps.data.unitOfWork.run(scope, async (tx) => {
+    const existing = (
+      await tx.of(documents).list({
+        where: [
+          { field: "collectionName", op: "eq", value: optIn.collectionName },
+          { field: "fileId", op: "eq", value: upload.fileId },
+        ],
+        limit: 1,
+      })
+    ).items[0]
+    return (
+      existing ??
+      (await tx.of(documents).insert({
+        collectionName: optIn.collectionName,
+        fileId: upload.fileId,
+        filename: file.filename,
+        status: "queued",
+        error: null,
+        indexedAt: null,
+        requestedBy: optIn.enabledBy,
+      }))
+    )
+  })
+  return indexPassages(deps, scope, row.id, options)
+}
+
+/** The upload's file, or undefined once it is no longer stored (deleted, or on its way out). */
+async function storedFile(deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string) {
+  try {
+    return await getFile(deps.files, scope, fileId)
+  } catch (error) {
+    if (error instanceof AppError && error.status === 404) return undefined
+    throw error
+  }
+}