510 lines
21 KiB
TypeScript
510 lines
21 KiB
TypeScript
// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
|
|
// Every subscriber accepts the delivery the worker's relay makes. The relay (`createOutboxRelay` of
|
|
// @conusai-kit/jobs-pgboss) runs each (record, subscriber) pair in the record's organization's scope with a system
|
|
// actor, and the worker binds each module's scope token to that delivery scope (`bindModulePorts`,
|
|
// apps/worker/src/ports.ts). A subscriber that works in a project builds the project from the record, comparing
|
|
// the record's organization with the delivery's.
|
|
//
|
|
// What runs here is the worker as it is deployed, on a disposable PostgreSQL with the three roles: the migrate job
|
|
// (`runAppMigrations`), the worker's runtime (`buildPostgresWorkerRuntime`: its data source, its pg-boss queue in
|
|
// the delivery's transaction), the worker's own port binding and pg-boss's worker (`createWorker`, its queues and
|
|
// its relay, driven a pass at a time), with the optional modules these events reach turned on the way an app
|
|
// turns them on. The platform is the fake one. A delivery that fails is
|
|
// a failure of this test, never a retry. Needs Docker.
|
|
|
|
import { execFileSync } from "node:child_process"
|
|
import { loadAppEnv } from "@app/config"
|
|
import { appModules } from "@app/modules"
|
|
import * as knowledge from "@app/modules/knowledge" // kit:knowledge
|
|
import { type Actor, createCaller, type ModuleManifest, newId, type TenantScope } from "@conusai-kit/core"
|
|
import {
|
|
beginDelete, // kit:knowledge
|
|
fileDeleted,
|
|
fileStored, // kit:knowledge
|
|
files, // kit:knowledge
|
|
filesDeps, // kit:knowledge
|
|
finishDeleteRecord, // kit:knowledge
|
|
platformBlobPathname, // kit:knowledge
|
|
} from "@conusai-kit/files"
|
|
import { runAppMigrations } from "@conusai-kit/host/migrate"
|
|
import { loadPostgresEnv, startObservability } from "@conusai-kit/host/postgres/config"
|
|
import { buildPostgresWorkerRuntime, type WorkerRuntime } from "@conusai-kit/host/worker"
|
|
import { createWorker, type Relay, type Worker, type WorkerLogEvent } from "@conusai-kit/jobs-pgboss"
|
|
import { createKnowledge } from "@conusai-kit/platform" // kit:knowledge
|
|
import { accountDataOf, createTenancyHooks, TENANCY, users } from "@conusai-kit/tenancy" // kit:knowledge
|
|
import { startFakePlatform } from "@conusai-kit/testing/fake-platform"
|
|
import { Pool } from "pg"
|
|
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"
|
|
import { bindModulePorts } from "../apps/worker/src/ports.js"
|
|
import { appSchedules } from "../apps/worker/src/schedules.js"
|
|
|
|
const APP_ROOT = new URL("..", import.meta.url).pathname
|
|
const docker = (...args: string[]): string =>
|
|
execFileSync("docker", args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim()
|
|
|
|
/** The optional modules these deliveries reach, turned on the way an app turns them on. */
|
|
const TURNED_ON: readonly ModuleManifest[] = [
|
|
knowledge.knowledgeModule, // kit:knowledge
|
|
]
|
|
|
|
/** The app's modules with those turned on. */
|
|
function selectedModules(): readonly ModuleManifest[] {
|
|
const on = new Set(TURNED_ON.map((module) => module.id))
|
|
return [...appModules.filter((module) => !on.has(module.id)), ...TURNED_ON]
|
|
}
|
|
|
|
let container = ""
|
|
let owner: Pool
|
|
let platform: Awaited<ReturnType<typeof startFakePlatform>>
|
|
let runtime: WorkerRuntime
|
|
let worker: Worker
|
|
let relay: Relay
|
|
/** Each failed delivery, as the relay reports it: the module and why. */
|
|
const failures: string[] = []
|
|
let project: TenantScope & { level: "project"; workspaceId: string; projectId: string; actor: Actor }
|
|
/** The project owner's caller, for what a person does in a test. */
|
|
let call: ReturnType<typeof createCaller>
|
|
|
|
beforeAll(async () => {
|
|
const name = `app-relay-scope-${crypto.randomUUID()}`
|
|
// The three roles a linked managed database provides, made by the same script the dev stack and e2e run.
|
|
container = docker(
|
|
"run",
|
|
"--detach",
|
|
"--rm",
|
|
"--name",
|
|
name,
|
|
"--label",
|
|
"conusai.test=app-relay-scope",
|
|
"--publish",
|
|
"127.0.0.1::5432",
|
|
"--env",
|
|
"POSTGRES_PASSWORD=postgres",
|
|
"--env",
|
|
"POSTGRES_DB=appdev",
|
|
"--volume",
|
|
`${APP_ROOT}dev:/conusai-dev:ro`,
|
|
"--entrypoint",
|
|
"sh",
|
|
"postgres:18",
|
|
"/conusai-dev/postgres-entrypoint.sh",
|
|
"postgres",
|
|
)
|
|
let port = ""
|
|
for (const deadline = Date.now() + 10_000; !/^[1-9][0-9]*$/.test(port); ) {
|
|
port = docker("port", container, "5432/tcp").split(":").at(-1) ?? ""
|
|
if (Date.now() > deadline) throw new Error("the test database's port was not published")
|
|
if (!/^[1-9][0-9]*$/.test(port)) await new Promise((resolve) => setTimeout(resolve, 100))
|
|
}
|
|
const url = (role: string): string => `postgresql://appdev_${role}:appdev_${role}@127.0.0.1:${port}/appdev`
|
|
const ownerUrl = url("owner")
|
|
// The init script creates the roles after the server first starts, then it restarts: ready is the owner's login.
|
|
owner = new Pool({ connectionString: ownerUrl, max: 1, connectionTimeoutMillis: 1_000 })
|
|
for (const deadline = Date.now() + 90_000; ; ) {
|
|
try {
|
|
await owner.query("select 1")
|
|
break
|
|
} catch (error) {
|
|
if (Date.now() > deadline) throw error
|
|
await new Promise((resolve) => setTimeout(resolve, 250))
|
|
}
|
|
}
|
|
|
|
platform = await startFakePlatform()
|
|
const source = {
|
|
NODE_ENV: "test",
|
|
CONUSAI_FAKE_PLATFORM: "1",
|
|
CONUSAI_API_URL: `${platform.url}/api`,
|
|
CONUSAI_API_TOKEN: "test-token",
|
|
CONUSAI_PROJECT_ID: "42",
|
|
CONUSAI_ORG_ID: "42",
|
|
CONUSAI_ORG_SLUG: "owner-org",
|
|
// The deployment's workspace, project and environment, which address the platform's knowledge service.
|
|
CONUSAI_WORKSPACE_SLUG: "general",
|
|
CONUSAI_PROJECT_SLUG: "app",
|
|
CONUSAI_ENVIRONMENT: "test",
|
|
POSTGRES_URL: url("app"),
|
|
POSTGRES_READER_URL: url("reader"),
|
|
}
|
|
const modules = selectedModules()
|
|
// The migrate job, as compose.yaml runs it: it reads the two logins it grants to from the environment.
|
|
const saved = { POSTGRES_URL: process.env.POSTGRES_URL, POSTGRES_READER_URL: process.env.POSTGRES_READER_URL }
|
|
Object.assign(process.env, { POSTGRES_URL: source.POSTGRES_URL, POSTGRES_READER_URL: source.POSTGRES_READER_URL })
|
|
try {
|
|
await runAppMigrations({ ownerUrl, modules, root: APP_ROOT })
|
|
} finally {
|
|
for (const [key, value] of Object.entries(saved)) {
|
|
if (value === undefined) delete process.env[key]
|
|
else process.env[key] = value
|
|
}
|
|
}
|
|
|
|
// The worker, as createPostgresWorker composes it for apps/worker/src/main.ts: its runtime, the app's port
|
|
// binding and pg-boss's worker, which creates the queues and serves them. Only the relay's timer is off: the
|
|
// test drives the worker's own relay a pass at a time.
|
|
loadAppEnv("worker", source)
|
|
const env = loadPostgresEnv("worker", source)
|
|
runtime = buildPostgresWorkerRuntime({ modules, env, observe: startObservability(env, { env: source }), source })
|
|
bindModulePorts(runtime.container, runtime.platform, runtime.registry)
|
|
// kit:knowledge:begin -- the knowledge port, bound as bindModulePorts binds it once an app has adopted knowledge
|
|
// (apps/worker/src/ports.ts `bindKnowledgePorts`). The shipped inclusion record has not adopted it; this test
|
|
// turns knowledge on by its manifest, so it binds the port the same way.
|
|
runtime.container.register(knowledge.KNOWLEDGE_SEARCH, (c) =>
|
|
knowledge.platformKnowledgeSearch(createKnowledge(runtime.platform.deployment), filesDeps(c), {
|
|
environment: runtime.platform.deployment.CONUSAI_ENVIRONMENT,
|
|
}),
|
|
)
|
|
// kit:knowledge:end
|
|
worker = createWorker({
|
|
registry: runtime.registry,
|
|
boss: runtime.boss,
|
|
claimGuard: runtime.claimGuard,
|
|
dataSource: runtime.data,
|
|
container: runtime.container,
|
|
schedules: appSchedules(),
|
|
relay: { enabled: false },
|
|
log: (event: WorkerLogEvent) => {
|
|
if (event.kind === "relay.failed") failures.push(`${event.event} -> ${event.subscriber}: ${String(event.error)}`)
|
|
},
|
|
})
|
|
await worker.start()
|
|
relay = worker.relay
|
|
|
|
// One organization with a project, made through the tenancy procedures as a person makes it.
|
|
const { data, registry } = runtime
|
|
const user = await data.global(users).insert({
|
|
issuer: "https://id.example.test",
|
|
subject: "ada@example.test",
|
|
email: "ada@example.test",
|
|
displayName: "Ada",
|
|
locale: "en",
|
|
})
|
|
const actor: Actor = { userId: user.id }
|
|
const tenancy = createTenancyHooks({ data, registry })
|
|
const requests = runtime.container.child()
|
|
requests.value(TENANCY, tenancy)
|
|
call = createCaller({ registry, container: requests, tenancy }, { actor })
|
|
const { orgId } = (await call.callByName("tenancy.org.create", { slug: "acme", name: "Acme" })) as { orgId: string }
|
|
const { workspaceId } = (await call.callByName("tenancy.workspace.create", {
|
|
org: "acme",
|
|
slug: "ops",
|
|
name: "Ops",
|
|
visibility: "org",
|
|
})) as { workspaceId: string }
|
|
const { projectId } = (await call.callByName("tenancy.project.create", {
|
|
org: "acme",
|
|
workspace: "ops",
|
|
slug: "site",
|
|
name: "Site",
|
|
moduleKey: "notes",
|
|
})) as { projectId: string }
|
|
project = { level: "project", actor, orgId, workspaceId, projectId }
|
|
// What the setup published is delivered first, so each test drains only its own records.
|
|
await relay.drain()
|
|
expect(failures).toEqual([])
|
|
}, 300_000)
|
|
|
|
afterAll(async () => {
|
|
await worker?.stop()
|
|
await runtime?.close()
|
|
await owner?.end()
|
|
await platform?.close()
|
|
if (container) docker("rm", "--force", "--volumes", container)
|
|
})
|
|
|
|
beforeEach(() => {
|
|
failures.length = 0
|
|
})
|
|
|
|
/** The jobs pg-boss holds for one queue, with the tenant ids each was enqueued in. */
|
|
async function queued(job: string): Promise<Array<{ scope: unknown; payload: unknown }>> {
|
|
const { rows } = await owner.query<{ data: { scope: unknown; payload: unknown } }>(
|
|
"select data from pgboss.job where name = $1 order by created_on",
|
|
[job],
|
|
)
|
|
return rows.map((row) => ({ scope: row.data.scope, payload: row.data.payload }))
|
|
}
|
|
|
|
describe("the worker's relay delivers to every subscriber in the organization's scope", () => {
|
|
it("delivers a file deletion to every module that subscribes to it", async () => {
|
|
const fileId = newId()
|
|
// kit:knowledge:begin -- the deleted file was indexed into a collection
|
|
await runtime.data.forScope(project).of(knowledge.documents).insert({
|
|
collectionName: "contracts",
|
|
fileId,
|
|
filename: "contract.pdf",
|
|
status: "indexed",
|
|
error: null,
|
|
indexedAt: "2026-11-09T00:00:00.000Z",
|
|
requestedBy: project.actor.userId,
|
|
})
|
|
// kit:knowledge:end
|
|
await runtime.data.unitOfWork.run(project, (tx) =>
|
|
tx.publish(fileDeleted, {
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
fileId,
|
|
storageKey: `files/${fileId}`,
|
|
size: 0,
|
|
}),
|
|
)
|
|
const pass = await relay.drain()
|
|
expect(failures).toEqual([])
|
|
expect(pass.delivered).toBeGreaterThan(0)
|
|
// kit:knowledge:begin -- knowledge cancelled the document and asked for its removal, which the relay delivers
|
|
const [document] = (await runtime.data.forScope(project).of(knowledge.documents).list({})).items
|
|
expect(document?.status).toBe("cancelled")
|
|
await relay.drain()
|
|
expect(failures).toEqual([])
|
|
expect(await queued(knowledge.REMOVE_JOB)).toMatchObject([
|
|
{
|
|
scope: {
|
|
level: "project",
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
},
|
|
payload: { fileId, collectionName: "contracts" },
|
|
},
|
|
])
|
|
// kit:knowledge:end
|
|
})
|
|
|
|
// kit:knowledge:begin
|
|
it("delivers knowledge's own index request, which enqueues the job in the record's project", async () => {
|
|
const documentId = newId()
|
|
await runtime.data.unitOfWork.run(project, (tx) =>
|
|
tx.publish(knowledge.documentIndexRequested, {
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
documentId,
|
|
collectionName: "contracts",
|
|
fileId: newId(),
|
|
}),
|
|
)
|
|
await relay.drain()
|
|
expect(failures).toEqual([])
|
|
expect(await queued(knowledge.INDEX_JOB)).toMatchObject([
|
|
{
|
|
scope: {
|
|
level: "project",
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
},
|
|
payload: { documentId },
|
|
},
|
|
])
|
|
})
|
|
// kit:knowledge:end
|
|
|
|
// kit:knowledge:begin
|
|
/** A stored file of the project, indexed into a collection: what an upload and an index leave behind. */
|
|
async function indexedFile(): Promise<{ fileId: string; storageKey: string }> {
|
|
const storageKey = `files/${newId()}`
|
|
const stored = await runtime.data
|
|
.forScope(project)
|
|
.of(files)
|
|
.insert({
|
|
status: "stored",
|
|
storageKey,
|
|
filename: "contract.pdf",
|
|
contentType: "application/pdf",
|
|
size: 1024,
|
|
// A stored row always has its size, digest and time (kit-files `viewOf` refuses one without them).
|
|
sha256: "0".repeat(64),
|
|
uploadedBy: project.actor.userId,
|
|
storedAt: "2026-11-09T00:00:00.000Z",
|
|
uploadId: null,
|
|
declaredSize: 1024,
|
|
partSize: null,
|
|
deletingAt: null,
|
|
})
|
|
const fileId = stored.id
|
|
await runtime.data.forScope(project).of(knowledge.documents).insert({
|
|
collectionName: "contracts",
|
|
fileId,
|
|
filename: "contract.pdf",
|
|
status: "indexed",
|
|
error: null,
|
|
indexedAt: "2026-11-09T00:00:00.000Z",
|
|
requestedBy: project.actor.userId,
|
|
})
|
|
return { fileId, storageKey }
|
|
}
|
|
|
|
/** The remove job's rows for one file, as pg-boss holds them. */
|
|
async function removeJob(fileId: string) {
|
|
const { rows } = await owner.query<{ state: string; retry_count: number }>(
|
|
"select state, retry_count from pgboss.job where name = $1 and data->'payload'->>'fileId' = $2",
|
|
[knowledge.REMOVE_JOB, fileId],
|
|
)
|
|
return rows
|
|
}
|
|
|
|
/** The platform knowledge service's `remove` calls for one stored object. */
|
|
async function removalsOf(storageKey: string) {
|
|
const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as {
|
|
knowledge: Array<{ route: string; documentRef?: string }>
|
|
}
|
|
return state.knowledge.filter((call) => call.route === "remove" && call.documentRef?.endsWith(storageKey))
|
|
}
|
|
|
|
it("removes a deleted file's indexed text from the knowledge service once, with the ref the file had (F407)", async () => {
|
|
const { fileId, storageKey } = await indexedFile()
|
|
// The files module's own deletion: the row is marked, then removed, and `FileDeleted` is published with it.
|
|
await beginDelete(filesDeps(runtime.container), project, fileId, project.actor.userId)
|
|
expect(await finishDeleteRecord(runtime.data, project, fileId)).toBe(true)
|
|
await relay.drain() // FileDeleted -> knowledge cancels the document and asks for its removal
|
|
await relay.drain() // documentRemoveRequested -> the remove job, which pg-boss's worker runs
|
|
expect(failures).toEqual([])
|
|
|
|
const removeJob = async () => {
|
|
const { rows } = await owner.query<{ state: string; retry_count: number }>(
|
|
"select state, retry_count from pgboss.job where name = $1 and data->'payload'->>'fileId' = $2",
|
|
[knowledge.REMOVE_JOB, fileId],
|
|
)
|
|
return rows
|
|
}
|
|
await expect.poll(async () => (await removeJob())[0]?.state, { timeout: 30_000 }).toBe("completed")
|
|
expect(await removeJob()).toEqual([{ state: "completed", retry_count: 0 }])
|
|
const state = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as {
|
|
knowledge: Array<{ route: string; documentRef?: string }>
|
|
}
|
|
const removals = state.knowledge.filter((call) => call.route === "remove")
|
|
expect(removals.filter((call) => call.documentRef?.endsWith(storageKey))).toEqual([
|
|
expect.objectContaining({ documentRef: platformBlobPathname(project, storageKey) }),
|
|
])
|
|
expect(
|
|
(await runtime.data.forScope(project).of(knowledge.documents).list({})).items.filter(
|
|
(row) => row.fileId === fileId,
|
|
),
|
|
).toEqual([])
|
|
|
|
// A second run of the same job, as a redelivery makes it, finds nothing left to remove and does nothing.
|
|
const { rows: delivered } = await owner.query<{ payload: unknown }>(
|
|
"select data->'payload' as payload from pgboss.job where name = $1 and data->'payload'->>'fileId' = $2",
|
|
[knowledge.REMOVE_JOB, fileId],
|
|
)
|
|
const inProject = runtime.container.child()
|
|
inProject.value(knowledge.KNOWLEDGE_SCOPE, project)
|
|
await runtime.registry.jobs[knowledge.REMOVE_JOB]?.def.handler(
|
|
delivered[0]?.payload as never,
|
|
{
|
|
container: inProject,
|
|
} as never,
|
|
)
|
|
const again = (await (await fetch(`${platform.url}/__e2e/state`)).json()) as typeof state
|
|
expect(
|
|
again.knowledge.filter((call) => call.route === "remove" && call.documentRef?.endsWith(storageKey)),
|
|
).toHaveLength(1)
|
|
})
|
|
|
|
it("removes it once too when a person's removal is still waiting as the file is deleted (F407, review R1)", async () => {
|
|
const { fileId, storageKey } = await indexedFile()
|
|
// A member removes the document: it is cancelled and its removal is published...
|
|
await call.callByName("knowledge.document.remove", {
|
|
org: "acme",
|
|
workspace: "ops",
|
|
project: "site",
|
|
collectionName: "contracts",
|
|
fileId,
|
|
})
|
|
// ...and the file is deleted before the relay has delivered that removal, so its row is gone when the job runs.
|
|
await beginDelete(filesDeps(runtime.container), project, fileId, project.actor.userId)
|
|
expect(await finishDeleteRecord(runtime.data, project, fileId)).toBe(true)
|
|
await relay.drain()
|
|
await relay.drain()
|
|
expect(failures).toEqual([])
|
|
await expect.poll(async () => (await removeJob(fileId))[0]?.state, { timeout: 30_000 }).toBe("completed")
|
|
expect(await removeJob(fileId)).toEqual([{ state: "completed", retry_count: 0 }])
|
|
expect(await removalsOf(storageKey)).toEqual([
|
|
expect.objectContaining({ documentRef: platformBlobPathname(project, storageKey) }),
|
|
])
|
|
expect(
|
|
(await runtime.data.forScope(project).of(knowledge.documents).list({})).items.filter(
|
|
(row) => row.fileId === fileId,
|
|
),
|
|
).toEqual([])
|
|
})
|
|
|
|
it("delivers a stored PDF to knowledge's files processor, which enqueues the automatic intake in the upload's project", async () => {
|
|
const fileId = newId()
|
|
const sha256 = "a".repeat(64)
|
|
await runtime.data.unitOfWork.run(project, (tx) =>
|
|
tx.publish(fileStored, {
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
fileId,
|
|
storageKey: `files/${project.workspaceId}/${project.projectId}/${fileId}`,
|
|
contentType: "application/pdf",
|
|
size: 10,
|
|
sha256,
|
|
uploadedBy: project.actor.userId,
|
|
recovered: false,
|
|
}),
|
|
)
|
|
await relay.drain()
|
|
expect(failures).toEqual([])
|
|
expect(await queued(knowledge.AUTO_INDEX_JOB)).toMatchObject([
|
|
{
|
|
scope: {
|
|
level: "project",
|
|
orgId: project.orgId,
|
|
workspaceId: project.workspaceId,
|
|
projectId: project.projectId,
|
|
},
|
|
payload: { fileId, contentType: "application/pdf", sha256 },
|
|
},
|
|
])
|
|
})
|
|
it("exports and erases intake attribution through the registered account share under the runtime role", async () => {
|
|
const share = accountDataOf(runtime.registry).find((entry) => entry.key === "knowledge")
|
|
if (!share?.export || !share.erase) throw new Error("Knowledge account share is missing")
|
|
const scopes: Array<typeof project> = []
|
|
for (const slug of ["account-own", "account-other"]) {
|
|
const { projectId } = (await call.callByName("tenancy.project.create", {
|
|
org: "acme",
|
|
workspace: "ops",
|
|
slug,
|
|
name: slug,
|
|
moduleKey: "notes",
|
|
})) as { projectId: string }
|
|
scopes.push({ ...project, projectId })
|
|
}
|
|
const own = scopes[0]
|
|
const other = scopes[1]
|
|
if (!own || !other) throw new Error("Missing account project fixtures")
|
|
const { autoIndex } = knowledge
|
|
for (const [scope, enabledBy] of [
|
|
[own, project.actor.userId],
|
|
[other, "another-person"],
|
|
] as const) {
|
|
await runtime.data.forScope(scope).of(autoIndex).insert({
|
|
collectionName: "contracts",
|
|
enabledBy,
|
|
enabledAt: "2026-11-11T00:00:00.000Z",
|
|
})
|
|
}
|
|
const context = { subject: project.actor.userId, container: runtime.container }
|
|
const exported = await share.export(context)
|
|
if (!exported) throw new Error("Knowledge account export is missing")
|
|
expect(exported.automaticIndexing).toEqual(
|
|
expect.arrayContaining([expect.objectContaining({ orgId: own.orgId, projectId: own.projectId })]),
|
|
)
|
|
expect(exported.automaticIndexing).not.toEqual(
|
|
expect.arrayContaining([expect.objectContaining({ projectId: other.projectId })]),
|
|
)
|
|
await share.erase(context)
|
|
await share.erase(context)
|
|
expect((await runtime.data.forScope(own).of(autoIndex).list()).items).toEqual([])
|
|
expect((await runtime.data.forScope(other).of(autoIndex).list()).items).toMatchObject([
|
|
{ enabledBy: "another-person" },
|
|
])
|
|
})
|
|
// kit:knowledge:end
|
|
})
|