281 lines
12 KiB
TypeScript
281 lines
12 KiB
TypeScript
// The managed search service: the platform's documents-index service through the kit-platform
|
|
// Knowledge client. Indexing and search run server-side there (the service owns chunking,
|
|
// embeddings and the vector store); this module supplies the job lifecycle and the citations it
|
|
// answers with.
|
|
//
|
|
// Contract notes (the service's real shapes): the index submission carries the content's SHA-256
|
|
// (computed here from the job's bytes) as its version assertion; a quota denial surfaces on the
|
|
// job, which the poll below reads as the job's typed error. Blob pathnames and returned citations
|
|
// resolve through the files module's scoped rows, so an opaque file id never becomes a pathname
|
|
// and a foreign or stale service reference never becomes a valid app hit.
|
|
|
|
import { createHash } from "node:crypto"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import {
|
|
type FilesDeps,
|
|
findFileStorageByKey,
|
|
getFileStorage,
|
|
platformBlobPathname,
|
|
platformBlobStorageKey,
|
|
} from "@conusai-kit/files"
|
|
import { KNOWLEDGE_LIMITS, type Knowledge, type KnowledgeClient, KnowledgeError } from "@conusai-kit/platform"
|
|
import type { KnowledgeHit, KnowledgePassageIndex, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js"
|
|
import { platformKnowledgeReadiness } from "./readiness.js"
|
|
import { permanentRefusal } from "./refusals.js"
|
|
|
|
/** The service's tenant partition of a project: a search of the project and its indexed documents agree on it. */
|
|
const tenantOf = (scope: KnowledgeScope): string =>
|
|
createHash("sha256")
|
|
.update(JSON.stringify([scope.orgId, scope.workspaceId, scope.projectId]))
|
|
.digest("hex")
|
|
|
|
/** A passage's reference: the file's own plus `#p{page}.{index}` (the automatic PDF intake, core-11). */
|
|
const passageRef = (documentRef: string, passage: { page: number; index: number }): string =>
|
|
`${documentRef}#p${passage.page}.${passage.index}`
|
|
|
|
const PASSAGE_REF = /^(.+)#p([1-9][0-9]*)\.(0|[1-9][0-9]*)$/
|
|
|
|
/**
|
|
* `environment` is the deployment's (`CONUSAI_ENVIRONMENT`): the service keeps a collection in one environment,
|
|
* and a deployment token reaches only its own.
|
|
*/
|
|
export function platformKnowledgeSearch(
|
|
client: Knowledge,
|
|
files: FilesDeps,
|
|
options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {},
|
|
): KnowledgeSearchService {
|
|
const environment = options.environment?.trim()
|
|
const configuredEnvironment = (): string => {
|
|
if (!environment)
|
|
throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing")
|
|
return environment
|
|
}
|
|
const unwrap = (error: unknown): never => {
|
|
// The client's errors are already typed by reason; a non-knowledge failure is unavailable.
|
|
if (error instanceof KnowledgeError) throw error
|
|
throw new KnowledgeError(
|
|
"runtime_unavailable",
|
|
error instanceof Error ? error.message : "the knowledge service failed",
|
|
true,
|
|
)
|
|
}
|
|
/** The same, for an index job: a refusal waiting cannot fix is recorded on the document, not retried. */
|
|
const unwrapJob = (error: unknown): never => {
|
|
if (error instanceof KnowledgeError) {
|
|
const permanent = permanentRefusal(error)
|
|
if (permanent !== undefined) throw Object.assign(error, { permanent })
|
|
}
|
|
return unwrap(error)
|
|
}
|
|
const documentRef = async (scope: KnowledgeScope, fileId: string): Promise<string> => {
|
|
const reference = await getFileStorage(files, scope, fileId)
|
|
return platformBlobPathname(scope, reference.storageKey)
|
|
}
|
|
|
|
return {
|
|
readiness(scope) {
|
|
if (!environment)
|
|
return Promise.resolve({
|
|
state: "unavailable",
|
|
reasons: ["CONUSAI_ENVIRONMENT is not set"],
|
|
refusal: "configuration_missing",
|
|
})
|
|
return platformKnowledgeReadiness(client).readiness(scope)
|
|
},
|
|
|
|
async index(scope, input) {
|
|
try {
|
|
// The service answers 404 for a collection it does not hold. An app's collection is only a row of
|
|
// this module (a write cannot leave the process), so the first index creates it; creating it again
|
|
// finds it.
|
|
const activeEnvironment = configuredEnvironment()
|
|
const reference = await documentRef(scope, input.fileId)
|
|
const signal = AbortSignal.any([
|
|
AbortSignal.timeout(options.timeoutMs ?? 300_000),
|
|
...(input.signal ? [input.signal] : []),
|
|
])
|
|
signal.throwIfAborted()
|
|
await client.createCollection({ collection: input.collectionName, environment: activeEnvironment }, { signal })
|
|
const { jobId } = await client.index(
|
|
{
|
|
collection: input.collectionName,
|
|
documentRef: reference,
|
|
options: {
|
|
visibility: "tenant",
|
|
tenant: tenantOf(scope),
|
|
contentSha256: createHash("sha256").update(input.bytes).digest("hex"),
|
|
},
|
|
},
|
|
{ signal },
|
|
)
|
|
// The service's job is durable; poll until it leaves the queue. A caller-side cancel is
|
|
// the job's own AbortSignal, checked between polls.
|
|
for (;;) {
|
|
signal.throwIfAborted()
|
|
const job = await client.job(jobId, { signal })
|
|
if (job.status === "succeeded") return
|
|
if (job.status === "failed")
|
|
throw new KnowledgeError(
|
|
job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable",
|
|
job.error ?? "the indexing job failed",
|
|
)
|
|
if (job.status === "cancelled")
|
|
throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled")
|
|
await delay(options.intervalMs ?? 1_000, undefined, { signal })
|
|
}
|
|
} catch (error) {
|
|
return unwrapJob(error)
|
|
}
|
|
},
|
|
|
|
async search(scope, input) {
|
|
try {
|
|
configuredEnvironment()
|
|
const results = await client.search({
|
|
collection: input.collectionName,
|
|
query: input.query,
|
|
limit: input.limit,
|
|
tenant: tenantOf(scope),
|
|
})
|
|
const hits: KnowledgeHit[] = []
|
|
for (const [at, hit] of results.entries()) {
|
|
// A passage of a PDF the intake indexed names its file's reference, and cites from its metadata:
|
|
// the service reports pages only for sources it parsed itself.
|
|
const passage = PASSAGE_REF.exec(hit.documentRef)
|
|
const storageKey = platformBlobStorageKey(scope, passage?.[1] ?? hit.documentRef)
|
|
if (storageKey === null) continue
|
|
const reference = await findFileStorageByKey(files, scope, storageKey)
|
|
if (reference === null) continue
|
|
const metadata = "metadata" in hit ? (hit.metadata as Readonly<Record<string, string>>) : {}
|
|
if (passage && metadata.file_id !== reference.file.fileId) continue
|
|
const page = passage ? Number(passage[2]) : hit.citations[0]?.page
|
|
const blockId = passage ? metadata.block_ids?.split(",")[0] || undefined : hit.citations[0]?.blockId
|
|
hits.push({
|
|
fileId: reference.file.fileId,
|
|
filename: reference.file.filename,
|
|
collectionName: input.collectionName,
|
|
score: hit.score,
|
|
chunk: at + 1,
|
|
snippet: hit.snippet,
|
|
...(page === undefined ? {} : { page }),
|
|
...(blockId === undefined ? {} : { blockId }),
|
|
...("chunkId" in hit && typeof hit.chunkId === "string" ? { chunkId: hit.chunkId } : {}),
|
|
})
|
|
}
|
|
return hits
|
|
} catch (error) {
|
|
// Nothing has been indexed into this collection yet, so the service has not heard of it.
|
|
if (error instanceof KnowledgeError && error.reason === "unavailable_resource") return []
|
|
return unwrap(error)
|
|
}
|
|
},
|
|
|
|
async remove(scope, input) {
|
|
try {
|
|
// Every removal asked for since F407 names its object; only a record from before it is looked up.
|
|
const reference =
|
|
input.storageKey === undefined
|
|
? await documentRef(scope, input.fileId)
|
|
: platformBlobPathname(scope, input.storageKey)
|
|
await client.remove(reference)
|
|
} catch (error) {
|
|
return unwrap(error)
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The automatic PDF intake's index (core-11): one inline item per passage, in the project's tenant partition,
|
|
* under the file's Blob reference plus `#p{page}.{index}`, with its page, its blocks and its file in metadata.
|
|
* Items go at most the service's count per call, and every item's durable job is waited for; a refusal waiting
|
|
* cannot fix is the document's, as with a whole file.
|
|
*/
|
|
export function platformPassageIndex(
|
|
client: Pick<KnowledgeClient, "createCollection" | "indexInline" | "job" | "remove">,
|
|
files: FilesDeps,
|
|
options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {},
|
|
): KnowledgePassageIndex {
|
|
const environment = options.environment?.trim()
|
|
const failure = (error: unknown): never => {
|
|
if (error instanceof KnowledgeError) {
|
|
const permanent = permanentRefusal(error)
|
|
if (permanent !== undefined) throw Object.assign(error, { permanent })
|
|
throw error
|
|
}
|
|
throw new KnowledgeError(
|
|
"runtime_unavailable",
|
|
error instanceof Error ? error.message : "the knowledge service failed",
|
|
true,
|
|
)
|
|
}
|
|
|
|
return {
|
|
async reference(scope, fileId) {
|
|
const stored = await getFileStorage(files, scope, fileId)
|
|
return platformBlobPathname(scope, stored.storageKey)
|
|
},
|
|
|
|
async index(scope, input) {
|
|
try {
|
|
if (!environment)
|
|
throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing")
|
|
const signal = AbortSignal.any([
|
|
AbortSignal.timeout(options.timeoutMs ?? 300_000),
|
|
...(input.signal ? [input.signal] : []),
|
|
])
|
|
await client.createCollection({ collection: input.collectionName, environment }, { signal })
|
|
const tenant = tenantOf(scope)
|
|
const jobs: string[] = []
|
|
for (let at = 0; at < input.passages.length; at += KNOWLEDGE_LIMITS.inlineItems) {
|
|
signal.throwIfAborted()
|
|
const accepted = await client.indexInline({
|
|
collection: input.collectionName,
|
|
items: input.passages.slice(at, at + KNOWLEDGE_LIMITS.inlineItems).map((passage) => ({
|
|
documentRef: passageRef(input.documentRef, passage),
|
|
text: passage.text,
|
|
visibility: "tenant" as const,
|
|
tenant,
|
|
metadata: {
|
|
page: String(passage.page),
|
|
...(passage.blockIds.length === 0 ? {} : { block_ids: passage.blockIds.join(",") }),
|
|
file_id: input.fileId,
|
|
},
|
|
})),
|
|
})
|
|
jobs.push(...accepted.map((job) => job.jobId))
|
|
}
|
|
for (const jobId of jobs) {
|
|
for (;;) {
|
|
signal.throwIfAborted()
|
|
const job = await client.job(jobId, { signal })
|
|
if (job.status === "succeeded") break
|
|
if (job.status === "failed")
|
|
throw new KnowledgeError(
|
|
job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable",
|
|
job.error ?? "the indexing job failed",
|
|
)
|
|
if (job.status === "cancelled")
|
|
throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled")
|
|
await delay(options.intervalMs ?? 1_000, undefined, { signal })
|
|
}
|
|
}
|
|
} catch (error) {
|
|
return failure(error)
|
|
}
|
|
},
|
|
|
|
async remove(_scope, input) {
|
|
for (const passage of input.passages) {
|
|
try {
|
|
await client.remove(passageRef(input.documentRef, passage))
|
|
} catch (error) {
|
|
// A passage the service never indexed (the run was refused before it) is already removed.
|
|
if (error instanceof KnowledgeError && error.reason === "unavailable_resource") continue
|
|
failure(error)
|
|
}
|
|
}
|
|
},
|
|
}
|
|
}
|