Add SeederApi seeding to the e2e suite (#24381)

* Add SeederApi seeding to the e2e suite

Ports the seeder framework from the playwright branch into e2e/, updated
for the current SeederApi: basic auth, the { urls } emergency access
invite response, and the full SingleUserScene request and result.

Each test gets a play whose id tags what it seeds and what its pages
create, and which is deleted when the test ends. Seeding tests are
skipped when no SeederApi account is configured.

* Harden e2e seeding configuration and cleanup

- Skip seeding tests against a deployed vault while the SeederApi is
  local, since it seeds a database that vault never reads.
- Read E2E_SEEDER_USERNAME and E2E_SEEDER_PASSWORD as a pair, failing
  on a partial set, and resolve the seeder URL the same way whatever
  supplied the account. Empty values count as unset.
- Set up `play` before the browser context so the page closes before
  its data is deleted.
- Assert the seeded user is the one signed in, through the automation
  driver's lock capability.
- Document that client-created data is only cleaned up when Api and
  Identity have play-id tracking enabled.

* Use Playwright's request client and routing for e2e seeding

- Call the SeederApi through an APIRequestContext opened per play, so
  credentials are read once, calls appear in traces, and an unreachable
  SeederApi says so.
- Tag requests to the vault with the play id through context routing
  instead of patching fetch, which also covers XHR.
- Record video for newPage contexts, which the config's video option
  does not reach.
- Simplify Play.mangle.

* Say up front when seeding e2e tests skip and when they fail
This commit is contained in:
Oscar Hinton
2026-11-09 09:04:11 +01:00
committed by GitHub
parent 1a8bdbfe62
commit e3d0bc5709
12 changed files with 554 additions and 7 deletions

View File

@@ -59,6 +59,58 @@ E2E_ACCOUNT=usdev-e2e npm run test:e2e:web
`E2E_EMAIL`, `E2E_PASSWORD` and `E2E_SERVER` take precedence over the file when all
three are set. CI uses them.
## Seeded data
Tests that need their own users, organizations or items create them through the server's
SeederApi (`util/SeederApi` in bitwarden/server) instead of the shared accounts above.
Run it beside the local server:
```
cd util/SeederApi && dotnet run # http://localhost:5047
```
Tests that seed are skipped, not failed, when no `[seeder]` account is configured, and when
a deployed vault is paired with a local SeederApi. With an account configured, the SeederApi
must be running: every test importing `test` from `src/seeder/fixtures.ts` fails without it,
including ones that seed nothing.
It takes a basic-auth account from its `seederSettings:accounts`. Give the suite the same one
in a `[seeder]` section:
```
[seeder]
username=e2e
password=...
url=http://localhost:5047
```
`url` is optional. `E2E_SEEDER_USERNAME` and `E2E_SEEDER_PASSWORD` take precedence over the
file, as a pair, and `E2E_SEEDER_URL` over `url`.
Data a test creates through the client, rather than seeds, is only deleted when the server
tracks it too. Add `"testPlayIdTrackingEnabled": true` to `globalSettings` in the server's
`dev/secrets.json`, rerun `dev/setup_secrets.ps1`, and restart Api and Identity. The SeederApi
enables it on its own.
Import `test` from `src/seeder/fixtures.ts`, which adds a `play` fixture holding the test's
data, and `newPage` for a second user's page:
```ts
const user = await play.scene(
new SingleUserScene({ email: "user@example.com", password: randomUUID() }),
);
await logInToWebVault(page, seededAccount(user));
```
The server mangles unique values such as emails; `scene.mangle("user@example.com")` returns the
stored one. Everything a play seeds, and with tracking on everything its pages create through
the client, is deleted when the test ends. `E2E_KEEP_SEEDED_DATA=1` keeps it and prints the command that
deletes it; the SeederApi also deletes play data older than a day.
Add a scene or query by subclassing `SceneTemplate` in `src/seeder/scenes/` or
`QueryTemplate` in `src/seeder/queries/`, named after the server's class and typed after its
`Request` and result.
## CI
`.github/workflows/test-e2e-usdev.yml` runs every suite against usdev nightly, and on

21
e2e/src/accounts.ts Normal file
View File

@@ -0,0 +1,21 @@
import { Page } from "@playwright/test";
import { automationDriver } from "./automation-driver";
const CAPABILITY = "lock";
/** An account the client knows, see `UserLockStatus` in libs/automation-driver. */
export type ClientAccount = {
userId: string;
email: string;
status: "LoggedOut" | "Locked" | "Unlocked";
};
type LockCapability = {
listUsers(): Promise<ClientAccount[]>;
};
/** Every account the client knows, with its lock status. */
export function clientAccounts(page: Page): Promise<ClientAccount[]> {
return automationDriver<LockCapability>(page, CAPABILITY).listUsers();
}

View File

@@ -15,6 +15,14 @@ const DEFAULT_ACCOUNT = "local-web";
*/
const DEFAULT_SECOND_ACCOUNT = "local-web-2";
/**
* Section holding the SeederApi's basic-auth account, one of its `seederSettings:accounts`.
*/
const SEEDER_SECTION = "seeder";
/** Where `dotnet run` serves the SeederApi — see util/SeederApi/Properties/launchSettings.json. */
const DEFAULT_SEEDER_URL = "http://localhost:5047";
export type Account = {
email: string;
password: string;
@@ -22,6 +30,13 @@ export type Account = {
server: string;
};
export type SeederAccount = {
/** SeederApi base URL, e.g. `http://localhost:5047`. */
url: string;
username: string;
password: string;
};
/**
* Parses the INI-style `.debug/credentials.txt`:
*
@@ -95,25 +110,72 @@ export function secondAccount(): Account {
}
function readAccount(name: string): Account {
let contents: string;
const section = readSection(name, ["email", "password", "server"]);
return { email: section.email, password: section.password, server: vaultUrl(section.server) };
}
/**
* Reads the SeederApi's basic-auth account from `E2E_SEEDER_USERNAME` and `E2E_SEEDER_PASSWORD`
* when set, otherwise from the `[seeder]` section. Its URL comes from `E2E_SEEDER_URL`, then the
* section's `url`, then the local default, wherever the account came from.
*/
export function seederAccount(): SeederAccount {
const { username, password } =
seederLoginFromEnv() ?? readSection(SEEDER_SECTION, ["username", "password"]);
const url =
process.env.E2E_SEEDER_URL || readSections()?.get(SEEDER_SECTION)?.url || DEFAULT_SEEDER_URL;
return { url, username, password };
}
/** Whether a SeederApi account is configured, which tests that seed data need. */
export function hasSeederAccount(): boolean {
return seederLoginFromEnv() != null || (readSections()?.has(SEEDER_SECTION) ?? false);
}
/** Both variables, or neither: like `fromEnv`, a partial set is a misconfigured job. */
function seederLoginFromEnv(): { username: string; password: string } | undefined {
const { E2E_SEEDER_USERNAME: username, E2E_SEEDER_PASSWORD: password } = process.env;
if (!username && !password) {
return undefined;
}
if (!username || !password) {
throw new Error("Set both E2E_SEEDER_USERNAME and E2E_SEEDER_PASSWORD, or neither.");
}
return { username, password };
}
/** The credentials file's sections, or undefined when there is no file, as on CI. */
function readSections(): Map<string, Record<string, string>> | undefined {
try {
contents = readFileSync(CREDENTIALS_FILE, "utf8");
return parse(readFileSync(CREDENTIALS_FILE, "utf8"));
} catch {
return undefined;
}
}
function readSection(name: string, required: string[]): Record<string, string> {
const sections = readSections();
if (sections == null) {
throw new Error(`No credentials at ${CREDENTIALS_FILE}. See e2e/README.md.`);
}
const section = parse(contents).get(name);
const section = sections.get(name);
if (section == null) {
throw new Error(`No [${name}] section in ${CREDENTIALS_FILE}.`);
}
for (const key of ["email", "password", "server"] as const) {
for (const key of required) {
if (!section[key]) {
throw new Error(`[${name}] in ${CREDENTIALS_FILE} is missing ${key}.`);
}
}
return { email: section.email, password: section.password, server: vaultUrl(section.server) };
return section;
}
/** Accepts a bare host, e.g. `vault.usdev.bitwarden.pw`, as its https URL. */

View File

@@ -14,10 +14,10 @@ const WEB_POST_LOGIN_URL = new RegExp(`#\\/vault|${SETUP_EXTENSION_URL.source}`)
const WEB_POST_LOGIN_TIMEOUT_MS = 60_000;
/** Logs in to the web vault and skips onboarding, leaving the page on the vault. */
export async function logInToWebVault(page: Page) {
export async function logInToWebVault(page: Page, targetAccount: Account = account()) {
await skipOnboardingDialogs(page);
await page.goto("/");
await new LoginPage(page).logIn();
await new LoginPage(page).logIn(targetAccount);
await page.waitForURL(WEB_POST_LOGIN_URL, { timeout: WEB_POST_LOGIN_TIMEOUT_MS });
await skipExtensionSetup(page);
}

View File

@@ -0,0 +1,86 @@
import { BrowserContext, Page, test as base } from "@playwright/test";
import { pruneVideo, videoDir } from "../artifacts";
import { hasSeederAccount, seederAccount } from "../credentials";
import { IS_DEV_SERVER, WEB_VAULT_URL } from "../server";
import { Play } from "./play";
import { PLAY_ID_HEADER, SeederApi } from "./seeder-api";
type SeederFixtures = {
/** This test's server-side data, deleted once the test ends. */
play: Play;
/** Opens a page in a fresh context, e.g. to sign in a second user beside `page`. */
newPage: () => Promise<Page>;
};
/**
* Tags the context's requests to the vault with the play id, so a server tracking play ids
* (see e2e/README.md) records what the client creates, such as an account registered through
* the UI, and `play.clean` deletes it. Routing disables the context's HTTP cache, which a
* fresh context has nothing in anyway.
*/
async function tagRequests(context: BrowserContext, playId: string) {
const origin = new URL(WEB_VAULT_URL).origin;
await context.route(
(url) => url.origin === origin,
(route) =>
route.continue({ headers: { ...route.request().headers(), [PLAY_ID_HEADER]: playId } }),
);
}
const LOOPBACK_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]);
/** Why this run cannot seed, if it cannot. */
function seedingUnavailable(): string | undefined {
// CI drives a deployed vault with no SeederApi in reach.
if (!hasSeederAccount()) {
return "No SeederApi account configured, see e2e/README.md.";
}
// A local SeederApi writes to the local database, where a deployed vault never looks.
if (!IS_DEV_SERVER && LOOPBACK_HOSTS.has(new URL(seederAccount().url).hostname)) {
return `A local SeederApi cannot seed ${WEB_VAULT_URL}; set E2E_SEEDER_URL to its own.`;
}
return undefined;
}
/** Web tests that seed data. */
export const test = base.extend<SeederFixtures>({
play: async ({}, use, testInfo) => {
const reason = seedingUnavailable();
testInfo.skip(reason != null, reason);
const seeder = await SeederApi.connect();
try {
const play = new Play(seeder);
await use(play);
await play.clean();
} finally {
await seeder.dispose();
}
},
context: async ({ play, context }, use) => {
await tagRequests(context, play.id);
await use(context);
},
newPage: async ({ play, browser }, use, testInfo) => {
const contexts: BrowserContext[] = [];
await use(async () => {
// The config's options reach this context, but its video setting does not.
const context = await browser.newContext({ recordVideo: { dir: videoDir(testInfo) } });
contexts.push(context);
await tagRequests(context, play.id);
return context.newPage();
});
await Promise.all(contexts.map((context) => context.close()));
pruneVideo(testInfo);
},
});

60
e2e/src/seeder/play.ts Normal file
View File

@@ -0,0 +1,60 @@
import { randomUUID } from "node:crypto";
import { QueryTemplate } from "./query";
import { Scene, SceneTemplate } from "./scene";
import { SeederApi } from "./seeder-api";
/** Keeps a run's data on the server, e.g. `E2E_KEEP_SEEDED_DATA=1`, to inspect it afterwards. */
const KEEP_SEEDED_DATA = process.env.E2E_KEEP_SEEDED_DATA != null;
/**
* One test's server-side data. Everything it seeds, and everything its pages create
* through the client, is tagged with its id and deleted together by `clean`.
*/
export class Play {
readonly id = randomUUID();
/** Same shape as the server's mangle prefix, so local and seeded values read alike. */
private readonly mangleId = this.id.slice(0, 8);
constructor(private readonly seeder: SeederApi) {}
/** Runs a scene template on the server. */
async scene<TArgs, TResult>(
template: SceneTemplate<TArgs, TResult>,
): Promise<Scene<TArgs, TResult>> {
const { result, mangleMap } = await this.seeder.seed<TArgs, TResult>(
this.id,
template.template,
template.args,
);
return new Scene(template.args, result, mangleMap);
}
/** Runs a read-only query template on the server. */
async query<TArgs, TResult>(template: QueryTemplate<TArgs, TResult>): Promise<TResult> {
return this.seeder.query<TArgs, TResult>(template.template, template.args);
}
/**
* Makes a value the test creates through the client unique to this play, mangling it
* the way the server mangles seeded ones. Seed data instead wherever a scene exists.
*/
mangle(value: string): string {
return `${this.mangleId}${value.includes("@") ? "+" : "-"}${value}`;
}
/** Deletes the play's data, unless the run keeps it. */
async clean(): Promise<void> {
if (KEEP_SEEDED_DATA) {
// eslint-disable-next-line no-console -- the only way to hand the command to the developer
console.warn(
`Kept seeded data for play ${this.id}. Delete it with:\n${this.seeder.destroyCommand(this.id)}`,
);
return;
}
await this.seeder.destroy(this.id);
}
}

View File

@@ -0,0 +1,18 @@
import { QueryTemplate } from "../query";
/** See `EmergencyAccessInviteQuery.Request` in the server's util/Seeder/Queries. */
type Args = {
/** The grantee's email, as stored, e.g. from `scene.mangle`. */
email: string;
};
/** See `EmergencyAccessInviteQuery.Response` in the server's util/Seeder/Queries. */
type Result = {
/** Hash-route paths accepting each invite, e.g. `/accept-emergency?id=...`. */
urls: string[];
};
/** The links in the emergency access invite emails sent to a grantee. */
export class EmergencyAccessInviteQuery extends QueryTemplate<Args, Result> {
readonly template = "EmergencyAccessInviteQuery";
}

13
e2e/src/seeder/query.ts Normal file
View File

@@ -0,0 +1,13 @@
/**
* Names a read-only server-side query and its arguments. Each subclass matches an `IQuery`
* in the server's util/Seeder/Queries, by class name. Queries read what the test cannot
* observe through a client, such as the links in an invite email.
*/
export abstract class QueryTemplate<TArgs, TResult> {
abstract readonly template: string;
/** Never read; it carries `TResult` so `Play.query` can infer the query's result type. */
protected readonly resultType?: TResult;
constructor(readonly args: TArgs) {}
}

29
e2e/src/seeder/scene.ts Normal file
View File

@@ -0,0 +1,29 @@
/**
* Names a server-side scene and the arguments to run it with. Each subclass matches an
* `IScene` in the server's util/Seeder/Scenes, by class name.
*/
export abstract class SceneTemplate<TArgs, TResult> {
abstract readonly template: string;
/** Never read; it carries `TResult` so `Play.scene` can infer the scene's result type. */
protected readonly resultType?: TResult;
constructor(readonly args: TArgs) {}
}
/** Data a scene template created on the server, removed when the test's play is cleaned. */
export class Scene<TArgs, TResult> {
constructor(
readonly args: TArgs,
readonly result: TResult,
private readonly mangleMap: Record<string, string | null>,
) {}
/**
* The value the server stored for a magic string in the arguments. The server mangles
* unique values such as emails, so parallel and repeated runs never collide.
*/
mangle(value: string): string {
return this.mangleMap[value] ?? value;
}
}

View File

@@ -0,0 +1,78 @@
import { Account } from "../../credentials";
import { WEB_VAULT_URL } from "../../server";
import { Scene, SceneTemplate } from "../scene";
/** The server's `Bit.Core.Enums.GatewayType`, which the SeederApi reads as numbers. */
export const GatewayType = Object.freeze({
Stripe: 0,
Braintree: 1,
AppStore: 2,
PlayStore: 3,
BitPay: 4,
PayPal: 5,
Bank: 6,
} as const);
export type GatewayType = (typeof GatewayType)[keyof typeof GatewayType];
/** See `SingleUserScene.Request` in the server's util/Seeder/Scenes/SingleUserScene.cs. */
export type SingleUserArgs = {
/** Mangled by the server; use an `example.com` address. */
email: string;
/** Not mangled. */
password: string;
/** Implied by `premium`. */
emailVerified?: boolean;
premium?: boolean;
/** Writes a self-hosted premium license; only a self-hosted SeederApi accepts it with `premium`. */
selfHosted?: boolean;
/** Billing gateway identity, so the user resembles a billed one. No Stripe calls are made. */
gateway?: GatewayType;
/** Refused by a self-hosted SeederApi. */
gatewayCustomerId?: string;
/** Refused by a self-hosted SeederApi. */
gatewaySubscriptionId?: string;
/**
* PBKDF2 iterations, 5,000 to 2,000,000; 5,000 when unset. The web vault raises anything
* under 600,000 on login, after which `kdfIterations` and `key` in the result are stale.
*/
kdfIterations?: number;
/** Enrolls the user in email two-factor, delivered to the mangled email. */
enrollInEmail2Fa?: boolean;
/** Stores an undecryptable private key, which the web vault's Data Recovery tool replaces. */
corruptPrivateKey?: boolean;
};
/** See `SingleUserSceneResult` in the server's util/Seeder/Scenes/SingleUserScene.cs. */
export type SingleUserResult = {
userId: string;
/** The KDF's name, e.g. `PBKDF2_SHA256`. */
kdf: string;
kdfIterations: number;
/** The user key, encrypted with the master key. */
key: string;
/** The user key, base64, e.g. for enrolling the user in account recovery. */
decryptedKeyB64: string;
publicKey: string;
privateKey: string;
apiKey: string;
/** Whether a self-hosted premium license was written. */
premiumLicenseWritten: boolean;
/** Why a self-hosted premium license could not be written. */
premiumLicenseWarning: string | null;
/** Set with `enrollInEmail2Fa`. */
twoFactorRecoveryCode: string | null;
};
/** A standalone user with a master password. */
export class SingleUserScene extends SceneTemplate<SingleUserArgs, SingleUserResult> {
readonly template = "SingleUserScene";
}
/** The account to log in as, with the email the server stored. */
export function seededAccount(scene: Scene<SingleUserArgs, SingleUserResult>): Account {
return {
email: scene.mangle(scene.args.email),
password: scene.args.password,
server: WEB_VAULT_URL,
};
}

View File

@@ -0,0 +1,100 @@
import { APIRequestContext, APIResponse, request } from "@playwright/test";
import { seederAccount } from "../credentials";
/** Header the server tags created entities with, so `DELETE /seed/{playId}` can remove them. */
export const PLAY_ID_HEADER = "x-play-id";
/**
* Body of a seed response, see `SceneResponseModel` in the server's
* util/SeederApi/Models/Response/SeedResponseModel.cs.
*/
export type SeedResponse<TResult> = {
/** Each requested magic string, e.g. an email, to what the server actually stored. */
mangleMap: Record<string, string | null>;
result: TResult;
};
/**
* The SeederApi, called from the test runner rather than the page: it needs no proxy in
* the web build, and its basic-auth account never reaches the browser.
*/
export class SeederApi {
private constructor(
private readonly api: APIRequestContext,
readonly url: string,
) {}
static async connect(): Promise<SeederApi> {
const { url, username, password } = seederAccount();
// A trailing slash, so relative paths resolve under the URL's own path.
const baseURL = `${url.replace(/\/$/, "")}/`;
const api = await request.newContext({
// Explicit, or the config's web vault URL would apply.
baseURL,
httpCredentials: { username, password, send: "always" },
});
return new SeederApi(api, baseURL);
}
/** Runs a scene template, tagging everything it creates with the play id. */
async seed<TArgs, TResult>(
playId: string,
template: string,
args: TArgs,
): Promise<SeedResponse<TResult>> {
const response = await this.send(() =>
this.api.post("seed", {
data: { template, arguments: args },
headers: { [PLAY_ID_HEADER]: playId },
}),
);
return response.json();
}
/** Runs a read-only query template. */
async query<TArgs, TResult>(template: string, args: TArgs): Promise<TResult> {
const response = await this.send(() =>
this.api.post("query", { data: { template, arguments: args } }),
);
return response.json();
}
/** Deletes everything tagged with the play id, seeded or created through the client. */
async destroy(playId: string): Promise<void> {
await this.send(() => this.api.delete(this.playPath(playId)));
}
/** The curl command that deletes a play's data by hand, for runs that keep it. */
destroyCommand(playId: string): string {
return `curl -u <seeder account> -X DELETE ${new URL(this.playPath(playId), this.url)}`;
}
async dispose(): Promise<void> {
await this.api.dispose();
}
private playPath(playId: string): string {
return `seed/${encodeURIComponent(playId)}`;
}
private async send(call: () => Promise<APIResponse>): Promise<APIResponse> {
let response: APIResponse;
try {
response = await call();
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
throw new Error(`Could not reach the SeederApi at ${this.url}. Is it running?\n${detail}`);
}
if (!response.ok()) {
// The SeederApi answers a failed scene with `{ error, details }`, which names the cause.
throw new Error(`${response.url()} failed: ${response.status()} ${await response.text()}`);
}
return response;
}
}

View File

@@ -0,0 +1,28 @@
import { randomUUID } from "node:crypto";
import { expect } from "@playwright/test";
import { clientAccounts } from "../../../src/accounts";
import { logInToWebVault } from "../../../src/login";
import { test } from "../../../src/seeder/fixtures";
import { seededAccount, SingleUserScene } from "../../../src/seeder/scenes/single-user.scene";
const VAULT_URL = /#\/vault/;
test("logs into the vault as a user seeded for the test", async ({ page, play }) => {
// Given a user who exists only for this test
const user = await play.scene(
new SingleUserScene({ email: "seeded-user@example.com", password: randomUUID() }),
);
const account = seededAccount(user);
// When the user logs in with their master password
await logInToWebVault(page, account);
// Then the user should land on their own vault
await expect(page).toHaveURL(VAULT_URL);
await expect
.poll(() => clientAccounts(page))
.toContainEqual(expect.objectContaining({ email: account.email, status: "Unlocked" }));
});