73 lines
2.9 KiB
TypeScript
73 lines
2.9 KiB
TypeScript
// The automatic PDF intake (template lesson core-11): what the worker runs for one uploaded file the
|
|
// files module handed to this module's processor. A project that opted in (`knowledge_auto_index`)
|
|
// gets the PDF indexed into its collection as a document like any other, requested by the
|
|
// administrator who opted in: the job runs for nobody else, so that administrator's admission is what
|
|
// each step leaving the process checks again (`beforeIndex`). Anything else reads nothing.
|
|
|
|
import { AppError } from "@conusai-kit/core"
|
|
import { getFile } from "@conusai-kit/files"
|
|
import { AUTO_INDEX_CONTENT_TYPES, autoIndex, type DocumentView, documents } from "../contract/index.js"
|
|
import { indexPassages } from "./index-job.js"
|
|
import { type KnowledgeDeps, type KnowledgeScope, knowledgeEnabled } from "./ports.js"
|
|
|
|
/** What the files module's processor job carries about the upload. */
|
|
export interface AutoIndexUpload {
|
|
fileId: string
|
|
contentType: string
|
|
sha256: string
|
|
}
|
|
|
|
/**
|
|
* Indexes one upload of an opted-in project, or answers null and reads nothing: a type it does not read,
|
|
* a project that is not opted in or has knowledge off, a file that is gone, or one whose digest is not
|
|
* the upload's. A redelivered job finds the document row and resumes it, so the work is done once.
|
|
*/
|
|
export async function autoIndexUpload(
|
|
deps: KnowledgeDeps,
|
|
scope: KnowledgeScope,
|
|
upload: AutoIndexUpload,
|
|
options: { signal?: AbortSignal; finalAttempt?: boolean } = {},
|
|
): Promise<DocumentView | null> {
|
|
if (!(AUTO_INDEX_CONTENT_TYPES as readonly string[]).includes(upload.contentType)) return null
|
|
const optIn = (await deps.data.forScope(scope).of(autoIndex).list({ limit: 1 })).items[0]
|
|
if (!optIn) return null
|
|
if (!(await knowledgeEnabled(deps, scope))) return null
|
|
const file = await storedFile(deps, scope, upload.fileId)
|
|
if (!file || file.sha256 !== upload.sha256) return null
|
|
|
|
const row = await deps.data.unitOfWork.run(scope, async (tx) => {
|
|
const existing = (
|
|
await tx.of(documents).list({
|
|
where: [
|
|
{ field: "collectionName", op: "eq", value: optIn.collectionName },
|
|
{ field: "fileId", op: "eq", value: upload.fileId },
|
|
],
|
|
limit: 1,
|
|
})
|
|
).items[0]
|
|
return (
|
|
existing ??
|
|
(await tx.of(documents).insert({
|
|
collectionName: optIn.collectionName,
|
|
fileId: upload.fileId,
|
|
filename: file.filename,
|
|
status: "queued",
|
|
error: null,
|
|
indexedAt: null,
|
|
requestedBy: optIn.enabledBy,
|
|
}))
|
|
)
|
|
})
|
|
return indexPassages(deps, scope, row.id, options)
|
|
}
|
|
|
|
/** The upload's file, or undefined once it is no longer stored (deleted, or on its way out). */
|
|
async function storedFile(deps: KnowledgeDeps, scope: KnowledgeScope, fileId: string) {
|
|
try {
|
|
return await getFile(deps.files, scope, fileId)
|
|
} catch (error) {
|
|
if (error instanceof AppError && error.status === 404) return undefined
|
|
throw error
|
|
}
|
|
}
|