// The managed search service: the platform's documents-index service through the kit-platform // Knowledge client. Indexing and search run server-side there (the service owns chunking, // embeddings and the vector store); this module supplies the job lifecycle and the citations it // answers with. // // Contract notes (the service's real shapes): the index submission carries the content's SHA-256 // (computed here from the job's bytes) as its version assertion; a quota denial surfaces on the // job, which the poll below reads as the job's typed error. Blob pathnames and returned citations // resolve through the files module's scoped rows, so an opaque file id never becomes a pathname // and a foreign or stale service reference never becomes a valid app hit. import { createHash } from "node:crypto" import { setTimeout as delay } from "node:timers/promises" import { type FilesDeps, findFileStorageByKey, getFileStorage, platformBlobPathname, platformBlobStorageKey, } from "@conusai-kit/files" import { KNOWLEDGE_LIMITS, type Knowledge, type KnowledgeClient, KnowledgeError } from "@conusai-kit/platform" import type { KnowledgeHit, KnowledgePassageIndex, KnowledgeScope, KnowledgeSearchService } from "../core/ports.js" import { platformKnowledgeReadiness } from "./readiness.js" import { permanentRefusal } from "./refusals.js" /** The service's tenant partition of a project: a search of the project and its indexed documents agree on it. */ const tenantOf = (scope: KnowledgeScope): string => createHash("sha256") .update(JSON.stringify([scope.orgId, scope.workspaceId, scope.projectId])) .digest("hex") /** A passage's reference: the file's own plus `#p{page}.{index}` (the automatic PDF intake, core-11). */ const passageRef = (documentRef: string, passage: { page: number; index: number }): string => `${documentRef}#p${passage.page}.${passage.index}` const PASSAGE_REF = /^(.+)#p([1-9][0-9]*)\.(0|[1-9][0-9]*)$/ /** * `environment` is the deployment's (`CONUSAI_ENVIRONMENT`): the service keeps a collection in one environment, * and a deployment token reaches only its own. */ export function platformKnowledgeSearch( client: Knowledge, files: FilesDeps, options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {}, ): KnowledgeSearchService { const environment = options.environment?.trim() const configuredEnvironment = (): string => { if (!environment) throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing") return environment } const unwrap = (error: unknown): never => { // The client's errors are already typed by reason; a non-knowledge failure is unavailable. if (error instanceof KnowledgeError) throw error throw new KnowledgeError( "runtime_unavailable", error instanceof Error ? error.message : "the knowledge service failed", true, ) } /** The same, for an index job: a refusal waiting cannot fix is recorded on the document, not retried. */ const unwrapJob = (error: unknown): never => { if (error instanceof KnowledgeError) { const permanent = permanentRefusal(error) if (permanent !== undefined) throw Object.assign(error, { permanent }) } return unwrap(error) } const documentRef = async (scope: KnowledgeScope, fileId: string): Promise => { const reference = await getFileStorage(files, scope, fileId) return platformBlobPathname(scope, reference.storageKey) } return { readiness(scope) { if (!environment) return Promise.resolve({ state: "unavailable", reasons: ["CONUSAI_ENVIRONMENT is not set"], refusal: "configuration_missing", }) return platformKnowledgeReadiness(client).readiness(scope) }, async index(scope, input) { try { // The service answers 404 for a collection it does not hold. An app's collection is only a row of // this module (a write cannot leave the process), so the first index creates it; creating it again // finds it. const activeEnvironment = configuredEnvironment() const reference = await documentRef(scope, input.fileId) const signal = AbortSignal.any([ AbortSignal.timeout(options.timeoutMs ?? 300_000), ...(input.signal ? [input.signal] : []), ]) signal.throwIfAborted() await client.createCollection({ collection: input.collectionName, environment: activeEnvironment }, { signal }) const { jobId } = await client.index( { collection: input.collectionName, documentRef: reference, options: { visibility: "tenant", tenant: tenantOf(scope), contentSha256: createHash("sha256").update(input.bytes).digest("hex"), }, }, { signal }, ) // The service's job is durable; poll until it leaves the queue. A caller-side cancel is // the job's own AbortSignal, checked between polls. for (;;) { signal.throwIfAborted() const job = await client.job(jobId, { signal }) if (job.status === "succeeded") return if (job.status === "failed") throw new KnowledgeError( job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable", job.error ?? "the indexing job failed", ) if (job.status === "cancelled") throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled") await delay(options.intervalMs ?? 1_000, undefined, { signal }) } } catch (error) { return unwrapJob(error) } }, async search(scope, input) { try { configuredEnvironment() const results = await client.search({ collection: input.collectionName, query: input.query, limit: input.limit, tenant: tenantOf(scope), }) const hits: KnowledgeHit[] = [] for (const [at, hit] of results.entries()) { // A passage of a PDF the intake indexed names its file's reference, and cites from its metadata: // the service reports pages only for sources it parsed itself. const passage = PASSAGE_REF.exec(hit.documentRef) const storageKey = platformBlobStorageKey(scope, passage?.[1] ?? hit.documentRef) if (storageKey === null) continue const reference = await findFileStorageByKey(files, scope, storageKey) if (reference === null) continue const metadata = "metadata" in hit ? (hit.metadata as Readonly>) : {} if (passage && metadata.file_id !== reference.file.fileId) continue const page = passage ? Number(passage[2]) : hit.citations[0]?.page const blockId = passage ? metadata.block_ids?.split(",")[0] || undefined : hit.citations[0]?.blockId hits.push({ fileId: reference.file.fileId, filename: reference.file.filename, collectionName: input.collectionName, score: hit.score, chunk: at + 1, snippet: hit.snippet, ...(page === undefined ? {} : { page }), ...(blockId === undefined ? {} : { blockId }), ...("chunkId" in hit && typeof hit.chunkId === "string" ? { chunkId: hit.chunkId } : {}), }) } return hits } catch (error) { // Nothing has been indexed into this collection yet, so the service has not heard of it. if (error instanceof KnowledgeError && error.reason === "unavailable_resource") return [] return unwrap(error) } }, async remove(scope, input) { try { // Every removal asked for since F407 names its object; only a record from before it is looked up. const reference = input.storageKey === undefined ? await documentRef(scope, input.fileId) : platformBlobPathname(scope, input.storageKey) await client.remove(reference) } catch (error) { return unwrap(error) } }, } } /** * The automatic PDF intake's index (core-11): one inline item per passage, in the project's tenant partition, * under the file's Blob reference plus `#p{page}.{index}`, with its page, its blocks and its file in metadata. * Items go at most the service's count per call, and every item's durable job is waited for; a refusal waiting * cannot fix is the document's, as with a whole file. */ export function platformPassageIndex( client: Pick, files: FilesDeps, options: { environment?: string | undefined; timeoutMs?: number; intervalMs?: number } = {}, ): KnowledgePassageIndex { const environment = options.environment?.trim() const failure = (error: unknown): never => { if (error instanceof KnowledgeError) { const permanent = permanentRefusal(error) if (permanent !== undefined) throw Object.assign(error, { permanent }) throw error } throw new KnowledgeError( "runtime_unavailable", error instanceof Error ? error.message : "the knowledge service failed", true, ) } return { async reference(scope, fileId) { const stored = await getFileStorage(files, scope, fileId) return platformBlobPathname(scope, stored.storageKey) }, async index(scope, input) { try { if (!environment) throw new KnowledgeError("configuration_missing", "CONUSAI_ENVIRONMENT is required for knowledge indexing") const signal = AbortSignal.any([ AbortSignal.timeout(options.timeoutMs ?? 300_000), ...(input.signal ? [input.signal] : []), ]) await client.createCollection({ collection: input.collectionName, environment }, { signal }) const tenant = tenantOf(scope) const jobs: string[] = [] for (let at = 0; at < input.passages.length; at += KNOWLEDGE_LIMITS.inlineItems) { signal.throwIfAborted() const accepted = await client.indexInline({ collection: input.collectionName, items: input.passages.slice(at, at + KNOWLEDGE_LIMITS.inlineItems).map((passage) => ({ documentRef: passageRef(input.documentRef, passage), text: passage.text, visibility: "tenant" as const, tenant, metadata: { page: String(passage.page), ...(passage.blockIds.length === 0 ? {} : { block_ids: passage.blockIds.join(",") }), file_id: input.fileId, }, })), }) jobs.push(...accepted.map((job) => job.jobId)) } for (const jobId of jobs) { for (;;) { signal.throwIfAborted() const job = await client.job(jobId, { signal }) if (job.status === "succeeded") break if (job.status === "failed") throw new KnowledgeError( job.error === "quota-exhausted" ? "quota_exhausted" : "runtime_unavailable", job.error ?? "the indexing job failed", ) if (job.status === "cancelled") throw new KnowledgeError("runtime_unavailable", "the indexing job was cancelled") await delay(options.intervalMs ?? 1_000, undefined, { signal }) } } } catch (error) { return failure(error) } }, async remove(_scope, input) { for (const passage of input.passages) { try { await client.remove(passageRef(input.documentRef, passage)) } catch (error) { // A passage the service never indexed (the run was refused before it) is already removed. if (error instanceof KnowledgeError && error.reason === "unavailable_resource") continue failure(error) } } }, } }