ConusAI vendor v0.1.0-beta.114

test/dialect/account.test.ts

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The account module's data-subject rights on the mariadb and mongodb profiles: the same store
+// (modules/account/adapters/store.ts) over the profile's real data source, with the kernel's and the
+// modules' committed dialect migrations applied first. A person's export holds their rows and nobody
+// else's, and their deletion erases them and leaves another person's account as it was.
+//
+// It needs the engines: ACCOUNT_TEST_MARIADB_URL (MariaDB LTS) and ACCOUNT_TEST_MONGODB_URL (MongoDB 8 as
+// a single-node replica set, AT-D15), each naming a database of its own that the test may fill. Without
+// them each block is skipped by name; `scripts/vendor/test-template.sh app --profile <name>` starts both.
+
+import { type DialectProfile, expectedDialectMigrations } from "@app/config"
+import { sessions } from "@conusai-kit/auth"
+import { buildRegistry } from "@conusai-kit/core"
+import { entitiesOf } from "@conusai-kit/data"
+import { KIT_JOBS_ENTITIES } from "@conusai-kit/jobs-pgboss"
+import { orgMemberships, users } from "@conusai-kit/tenancy"
+import { afterAll, beforeAll, describe, expect, it } from "vitest"
+import type { DialectData } from "../../apps/worker/src/dialect/engine.js"
+import { engineFor } from "../../apps/worker/src/dialect/runtime.js"
+import { makeApp, type Person } from "../../modules/account/__tests__/harness.js"
+import type { AccountExport, DeletionView } from "../../modules/account/index.js"
+import { accountDeletions } from "../../modules/account/index.js"
+import { freshDialectModules } from "./fixture.js"
+
+const ENGINES: readonly [DialectProfile, string, string | undefined][] = [
+  ["mariadb", "MARIADB_URL", process.env.ACCOUNT_TEST_MARIADB_URL],
+  ["mongodb", "MONGODB_URL", process.env.ACCOUNT_TEST_MONGODB_URL],
+]
+
+/** What the migrate job of the profile applies (apps/worker/src/dialect/), on the engine under test. */
+async function migrate(
+  profile: DialectProfile,
+  env: Record<string, string>,
+  registry: ReturnType<typeof buildRegistry>,
+): Promise<string[]> {
+  const outcome = await engineFor(profile).migrate(env, registry)
+  return [...outcome.applied, ...outcome.skipped]
+}
+
+async function orgOf(person: Person, slug: string): Promise<string> {
+  const created = (await person.call().callByName("tenancy.org.create", { slug, name: slug })) as { orgId: string }
+  return created.orgId
+}
+
+for (const [profile, variable, url] of ENGINES) {
+  const modules = freshDialectModules(profile)
+  const registry = buildRegistry(modules)
+  describe.skipIf(!url)(`account data-subject rights on ${profile} (${variable})`, () => {
+    let data: DialectData
+    const env = { [variable]: url ?? "" }
+
+    beforeAll(async () => {
+      // The migrate job's whole list applies on the engine, and it is the list readiness expects.
+      expect((await migrate(profile, env, registry)).sort()).toEqual(
+        (await expectedDialectMigrations(registry, profile)).sort(),
+      )
+      const entities = [...entitiesOf(registry, profile), ...KIT_JOBS_ENTITIES]
+      expect(entities.map(({ table }) => table)).toEqual(
+        expect.arrayContaining(["account_deletion", "account_session_revocation"]),
+      )
+      data = await engineFor(profile).connect(env, entities)
+    }, 120_000)
+
+    afterAll(async () => {
+      await data?.close()
+    })
+
+    it("exports the person's own rows and deletes the account, leaving another person's as it was", async () => {
+      const app = makeApp({ data, modules })
+      const tag = Math.random().toString(36).slice(2, 8)
+      const ada = await app.signUp(`ada-${tag}@example.test`)
+      const grace = await app.signUp(`grace-${tag}@example.test`)
+      const adaOrg = await orgOf(ada, `solo-${tag}`)
+      await orgOf(grace, `other-${tag}`)
+
+      const exported = (await ada.call().callByName("account.data.export", {})) as AccountExport
+      expect(exported).toMatchObject({ subject: ada.actor.userId, complete: true, failed: [] })
+      expect(exported.sections.account).toMatchObject({
+        status: "ok",
+        data: { profile: { userId: ada.actor.userId, email: ada.email }, organizations: [{ slug: `solo-${tag}` }] },
+      })
+      // Negative control: nothing of the other person is in it.
+      expect(JSON.stringify(exported)).not.toContain(grace.email)
+      expect(JSON.stringify(exported)).not.toContain(`other-${tag}`)
+
+      const view = (await ada.call().callByName("account.deletion.request", { confirm: ada.email })) as DeletionView
+      expect(view).toMatchObject({ state: "completed", pending: [], abandoned: [] })
+
+      expect(await app.data.global(users).get(ada.actor.userId)).toBeNull()
+      expect(
+        await app.data.forScope({ level: "org", orgId: adaOrg, actor: ada.actor }).of(orgMemberships).count(),
+      ).toBe(0)
+      const adaSessions = (
+        await app.data.global(sessions).list({ where: [{ field: "userId", op: "eq", value: ada.actor.userId }] })
+      ).items
+      expect(adaSessions).toEqual([])
+      const ledger = (
+        await app.data
+          .global(accountDeletions)
+          .list({ where: [{ field: "userId", op: "eq", value: ada.actor.userId }] })
+      ).items
+      expect(ledger).toMatchObject([{ state: "completed" }])
+      expect(JSON.stringify(ledger)).not.toContain(ada.email)
+
+      // The other person keeps everything: their user row, their membership and their export.
+      expect((await app.data.global(users).get(grace.actor.userId))?.email).toBe(grace.email)
+      const graceExport = (await grace.call().callByName("account.data.export", {})) as AccountExport
+      expect(graceExport.sections.account).toMatchObject({
+        status: "ok",
+        data: { organizations: [{ slug: `other-${tag}` }] },
+      })
+    }, 60_000)
+  })
+}

test/dialect/composition.test.ts

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The mariadb and mongodb compositions (profiles/<name>/README.md): the migrate job applies exactly
+// the list the web shell's readiness probe expects, the job adapter's own table among them, and the
+// worker serves every module job on BullMQ. `kit template:prune --profile postgres` removes this folder.
+
+import { expectedDialectMigrations } from "@app/config"
+import { appModules } from "@app/modules"
+import { buildRegistry } from "@conusai-kit/core"
+import { describe, expect, it } from "vitest"
+import { mariadbJobsMigration } from "../../apps/worker/src/dialect/mariadb/index.js"
+import { mongodbJobsCommands } from "../../apps/worker/src/dialect/mongodb/index.js"
+import { bullmqJobs, dialectMigrations, queueSlug } from "../../apps/worker/src/dialect/runtime.js"
+import { dialectSchedules } from "../../apps/worker/src/dialect/schedules.js"
+
+const registry = buildRegistry([...appModules])
+
+describe("the dialect profiles' migrations", () => {
+  for (const profile of ["mariadb", "mongodb"] as const) {
+    it(`${profile}: the migrate job applies what readiness expects, kernel, jobs table and modules`, async () => {
+      const expected = await expectedDialectMigrations(registry, profile)
+      const applied = (await dialectMigrations(registry, profile)).map((migration) => migration.name)
+      expect([...applied].sort()).toEqual([...expected].sort())
+      expect(expected).toEqual(
+        expect.arrayContaining([
+          "0001_kernel",
+          "0007_kit-jobs",
+          "notes/0001_notes", // kit:notes
+          "account/0001_account",
+        ]),
+      )
+    })
+  }
+
+  it("builds the outbox relay's ledger table in each dialect", () => {
+    expect(mariadbJobsMigration().statements.join("\n")).toContain("create table `kit_processed_events`")
+    expect(mongodbJobsCommands()).toContainEqual(expect.objectContaining({ createCollection: "kit_processed_events" }))
+  })
+})
+
+describe("the dialect profiles' worker", () => {
+  it("serves every module job of the registry, with its own payload schema", () => {
+    const jobs = bullmqJobs(registry)
+    expect(Object.keys(jobs).sort()).toEqual(Object.keys(registry.jobs).sort())
+    for (const [name, def] of Object.entries(jobs)) expect(def.payload).toBe(registry.jobs[name]?.def.payload)
+  })
+
+  it("schedules on each dialect worker only the module jobs of modules installed on its profile", () => {
+    for (const profile of ["mariadb", "mongodb"] as const) {
+      const jobs = dialectSchedules(registry, profile)
+        .map((schedule) => schedule.job)
+        .sort()
+      // The account, file and inbox sweeps run on both; the agent module ships Postgres only.
+      expect(jobs).toEqual([
+        "account.deletion.sweep",
+        "account.sessions.revoke",
+        "files.recover",
+        "notifications.retention.sweep",
+      ])
+    }
+  })
+
+  it("keys the queue by the platform's project, in the alphabet BullMQ prefixes take", () => {
+    const env = (id: string) => ({ deployment: { CONUSAI_PROJECT_ID: id } }) as Parameters<typeof queueSlug>[0]
+    expect(queueSlug(env("42"))).toBe("p-42")
+    expect(queueSlug(env("Proj_ID.9"))).toMatch(/^[a-z0-9][a-z0-9-]*$/)
+  })
+})

test/dialect/fixture.ts

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// Fresh, synthetic backend-conformance fixtures: no existing app, record or
+// adoption history is rewritten. Only modules supporting the tested profile
+// are installed in this new fixture's registry and its matching data source.
+import type { DialectProfile } from "@app/config"
+import { appModules } from "@app/modules"
+
+export function freshDialectModules(profile: DialectProfile) {
+  return appModules.filter((module) => module.profiles === undefined || module.profiles.includes(profile))
+}

test/dialect/profile-registry.test.ts

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+import { type AppEnv, loadAppEnv, profileRegistry, type ServerObservability } from "@app/config"
+import { appModules, moduleInclusion } from "@app/modules"
+import { buildRegistry } from "@conusai-kit/core"
+import { createMemoryDataSource, entitiesOf } from "@conusai-kit/data"
+import { users } from "@conusai-kit/tenancy"
+import { afterEach, describe, expect, it, vi } from "vitest"
+import { buildDialectWorkerRuntime, engineFor, migrateDialect } from "../../apps/worker/src/dialect/runtime.js"
+import { makeApp } from "../../modules/account/__tests__/harness.js"
+import type { AccountExport, DeletionView } from "../../modules/account/index.js"
+import { freshDialectModules } from "./fixture.js"
+
+afterEach(() => vi.restoreAllMocks())
+
+describe.each(["mariadb", "mongodb"] as const)("%s module/profile composition", (profile) => {
+  it("names unsupported adopted modules and preserves their custody registry", () => {
+    const before = JSON.stringify(moduleInclusion)
+    expect(() => profileRegistry(appModules, profile)).toThrow(/agent, assistant/)
+    expect(JSON.stringify(moduleInclusion)).toBe(before)
+    expect(buildRegistry([...appModules]).modules.map(({ id }) => id)).toContain("assistant")
+  })
+
+  it("uses the same compatible fresh registry for data and account rights without changing starter history", async () => {
+    const before = JSON.stringify(moduleInclusion)
+    const modules = freshDialectModules(profile)
+    const registry = profileRegistry(modules, profile)
+    const data = createMemoryDataSource({ entities: entitiesOf(registry, profile) })
+    const app = makeApp({ data, modules })
+    expect(app.registry.modules.map(({ id }) => id)).toEqual(registry.modules.map(({ id }) => id))
+    const ada = await app.signUp("profile-ada@example.test")
+    const grace = await app.signUp("profile-grace@example.test")
+    await ada.call().callByName("tenancy.org.create", { slug: "profile-ada", name: "Ada fixture" })
+    await grace.call().callByName("tenancy.org.create", { slug: "profile-grace", name: "Grace fixture" })
+    const exported = (await ada.call().callByName("account.data.export", {})) as AccountExport
+    expect(exported).toMatchObject({ subject: ada.actor.userId, complete: true, failed: [] })
+    expect(JSON.stringify(exported)).not.toContain(grace.email)
+    const deleted = (await ada.call().callByName("account.deletion.request", { confirm: ada.email })) as DeletionView
+    expect(deleted).toMatchObject({ state: "completed", pending: [], abandoned: [] })
+    expect(await data.global(users).get(ada.actor.userId)).toBeNull()
+    expect((await data.global(users).get(grace.actor.userId))?.email).toBe(grace.email)
+    expect(JSON.stringify(moduleInclusion)).toBe(before)
+  })
+
+  it("refuses the actual worker and migrate paths before connecting or applying migrations", async () => {
+    const databaseUrl = `${profile === "mariadb" ? "mysql" : "mongodb"}://127.0.0.1:1/fixture`
+    const raw: Record<string, string> = {
+      NODE_ENV: "test",
+      CONUSAI_API_URL: "https://fixture.example.test/api",
+      CONUSAI_API_TOKEN: "synthetic-token",
+      CONUSAI_PROJECT_ID: "profile-refusal",
+      REDIS_URL: "redis://127.0.0.1:1",
+      [profile === "mariadb" ? "MARIADB_URL" : "MONGODB_URL"]:
+        `${profile === "mariadb" ? "mysql" : "mongodb"}://127.0.0.1:1/fixture`,
+    }
+    if (profile === "mariadb") raw.DATABASE_URL = databaseUrl
+    const env: AppEnv = loadAppEnv("worker", raw, profile)
+    const engine = engineFor(profile)
+    const connect = vi
+      .spyOn(engine, "connect")
+      .mockRejectedValue(new Error("database access forbidden in refusal control"))
+    const migrate = vi.spyOn(engine, "migrate").mockRejectedValue(new Error("migration forbidden in refusal control"))
+    await expect(buildDialectWorkerRuntime(env, {} as ServerObservability)).rejects.toThrow(/agent, assistant/)
+    expect(() => migrateDialect(env)).toThrow(/agent, assistant/)
+    expect(connect).not.toHaveBeenCalled()
+    expect(migrate).not.toHaveBeenCalled()
+  })
+})

test/dialect/stores.test.ts

@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: LicenseRef-ConusAI-Template-1.0
+// The web shell's stores of the mariadb and mongodb profiles (apps/web/src/server/dialect/) on the
+// real engines: the readiness probes answer, the migration level the migrate job wrote is what
+// readiness expects, the MongoDB store refuses a standalone server at boot (AT-D15), and the BullMQ
+// queue enqueues on Valkey under the app's own prefix.
+//
+// It needs ACCOUNT_TEST_MARIADB_URL, ACCOUNT_TEST_MONGODB_URL (a replica set) and ACCOUNT_TEST_REDIS_URL,
+// the databases test/dialect/account.test.ts migrated; without them each block is skipped by name.
+
+import { randomUUID } from "node:crypto"
+import { type AppEnv, expectedDialectMigrations, liveCaptureMigrations, loadAppEnv } from "@app/config"
+import { notes } from "@app/modules/notes"
+import { buildRegistry } from "@conusai-kit/core"
+import { entitiesOf } from "@conusai-kit/data"
+import { KIT_JOBS_ENTITIES } from "@conusai-kit/jobs-pgboss"
+import { Redis } from "ioredis"
+import { describe, expect, it } from "vitest"
+import { dialectStores, queueSlug } from "../../apps/web/src/server/dialect/stores"
+import { engineFor } from "../../apps/worker/src/dialect/runtime.js"
+import { freshDialectModules } from "./fixture.js"
+
+const redisUrl = process.env.ACCOUNT_TEST_REDIS_URL
+const platform = {
+  NODE_ENV: "test",
+  CONUSAI_API_URL: "https://api.example.test/api",
+  CONUSAI_API_TOKEN: "token",
+  CONUSAI_PROJECT_ID: "w06-stores",
+}
+
+const ENGINES = [
+  [
+    "mariadb",
+    { MARIADB_URL: process.env.ACCOUNT_TEST_MARIADB_URL, DATABASE_URL: process.env.ACCOUNT_TEST_MARIADB_URL },
+  ],
+  ["mongodb", { MONGODB_URL: process.env.ACCOUNT_TEST_MONGODB_URL }],
+] as const
+
+for (const [profile, store] of ENGINES) {
+  const registry = buildRegistry(freshDialectModules(profile))
+  const url = Object.values(store)[0]
+  describe.skipIf(!url || !redisUrl)(`the web shell's ${profile} store`, () => {
+    it("answers readiness at the migrate job's level and enqueues on its own BullMQ prefix", async () => {
+      const raw = { ...platform, ...store, REDIS_URL: redisUrl }
+      await engineFor(profile).migrate(raw, registry)
+      const env: AppEnv = loadAppEnv("web", raw, profile)
+      const stores = dialectStores(
+        env,
+        [...entitiesOf(registry, profile), ...KIT_JOBS_ENTITIES],
+        liveCaptureMigrations(registry, profile),
+      )
+      if (!stores) throw new Error("no dialect store")
+      try {
+        await stores.probes.boot()
+        await stores.probes.store()
+        const applied = await stores.probes.applied()
+        for (const name of await expectedDialectMigrations(registry, profile))
+          expect(applied.has(name), name).toBe(true)
+        // A request's enqueue reaches Valkey under this app's prefix.
+        const id = await stores
+          .queue()
+          .enqueue(
+            { level: "org", orgId: "00000000-0000-4000-8000-000000000001", actor: { userId: "u" } },
+            "files.recover",
+            { probe: true },
+          )
+        expect(id).toBeTruthy()
+        const redis = new Redis(redisUrl as string)
+        try {
+          const keys = await redis.keys(`kit-jobs-${queueSlug(env)}*`)
+          expect(keys.length).toBeGreaterThan(0)
+        } finally {
+          await redis.quit()
+        }
+        // The live log: the migrate job prepared notes_note (MariaDB row triggers, MongoDB pre-images),
+        // so a note written through the repository is in the log, ids only, in its own organization.
+        {
+          const log = stores.liveLog?.()
+          if (!log) throw new Error(`the ${profile} store has no live log`)
+          await log.start()
+          const scope = {
+            level: "project",
+            orgId: randomUUID(),
+            workspaceId: randomUUID(),
+            projectId: randomUUID(),
+            actor: { userId: randomUUID() },
+          } as const
+          const since = profile === "mongodb" ? (await log.head(scope.orgId)).position : { txid: 0n, seq: 0n }
+          const note = await stores.data
+            .forScope(scope)
+            .of(notes)
+            .insert({ title: "live", body: "b", createdBy: scope.actor.userId })
+          let rows: Awaited<ReturnType<typeof log.read>>["final"] = []
+          for (let attempt = 0; attempt < 50 && rows.length === 0; attempt++) {
+            const read = await log.read(scope.orgId, since, 10)
+            rows = [...read.final, ...read.early]
+            if (rows.length === 0) await new Promise((resolve) => setTimeout(resolve, 100))
+          }
+          expect(rows.map((row) => [row.entity, row.id, row.op, row.scope.ws])).toEqual([
+            ["notes.note", note.id, "u", scope.workspaceId],
+          ])
+          expect((await log.read(randomUUID(), since, 10)).final).toEqual([])
+          await log.close()
+        }
+      } finally {
+        await stores.close()
+      }
+    }, 60_000)
+  })
+}
+
+describe.skipIf(!process.env.ACCOUNT_TEST_MONGODB_STANDALONE_URL)(
+  "the web shell's mongodb store on a standalone server",
+  () => {
+    it("refuses to boot with the typed replica_set error (AT-D15)", async () => {
+      const env = loadAppEnv(
+        "web",
+        { ...platform, MONGODB_URL: process.env.ACCOUNT_TEST_MONGODB_STANDALONE_URL, REDIS_URL: "redis://127.0.0.1:1" },
+        "mongodb",
+      )
+      const registry = buildRegistry(freshDialectModules("mongodb"))
+      const stores = dialectStores(env, entitiesOf(registry, "mongodb"))
+      if (!stores) throw new Error("no dialect store")
+      try {
+        await expect(stores.probes.boot()).rejects.toMatchObject({ name: "MongoTopologyError", setting: "replica_set" })
+      } finally {
+        await stores.close()
+      }
+    }, 60_000)
+  },
+)