openhands
8ee144745a
The port-conflict guard added in the previous commit made the six existing blue/green deployment simulation tests fail. assert_port_free() shells out to ss, and the simulation harness stubs git, curl, docker, nginx, pnpm and node — but not ss. Because the runner is self-hosted and the containers use --net=host, the simulation saw the production CMS containers holding 3002 and 3003 and refused to start its own candidate. The harness now stubs ss. It reports no listener for every scenario except 'port-taken', which reserves whichever port the script asks about, so the simulation stays independent of the host it runs on. Also switched the ss probe from `command -v ss` to `type ss`. The stub is a shell function delivered through BASH_ENV; `command -v` happens to find it, but `type` is the reliable test for "is this resolvable", and the two differ across shells. Added a regression test for the guard itself: with the candidate port already occupied, the deploy must fail, must not have run `docker run`, and must leave the nginx upstream untouched on the old port — no half-finished cutover. Verified it fails when the assert_port_free call is removed. Deploy simulation: 26 passed. Full unit suite: 3316 passed, 12 skipped.
361 lines
13 KiB
TypeScript
361 lines
13 KiB
TypeScript
// @ts-nocheck
|
|
import { spawnSync } from "node:child_process";
|
|
import {
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { delimiter, dirname, join, resolve } from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
const root = process.cwd();
|
|
const bash =
|
|
process.platform === "win32"
|
|
? ((process.env.PATH ?? "")
|
|
.split(delimiter)
|
|
.flatMap((dir) => [
|
|
join(dir, "bash.exe"),
|
|
join(dirname(dir), "bin", "bash.exe"),
|
|
join(dirname(dirname(dir)), "bin", "bash.exe"),
|
|
])
|
|
.find((path) => existsSync(path)) ?? "bash")
|
|
: "bash";
|
|
const sha = "a".repeat(40);
|
|
function simulate(
|
|
scenario: string,
|
|
previous = "epicnext-cms-app",
|
|
both = false,
|
|
) {
|
|
const dir = mkdtempSync(join(tmpdir(), "cms-deploy-test-"));
|
|
try {
|
|
mkdirSync(join(dir, "production"));
|
|
// Een werkende deploymap heeft een DATABASE_URL; die gebruikt
|
|
// scripts/ci-deploy.sh voor de migraties op de host.
|
|
writeFileSync(
|
|
join(dir, "production", ".env"),
|
|
'HOTEL_NAME="Test Hotel"\nDATABASE_URL="mysql://test:[email protected]:3306/test"\n',
|
|
);
|
|
if (previous) writeFileSync(join(dir, previous), "old\n");
|
|
if (both) writeFileSync(join(dir, "epicnext-cms-app"), "old\n");
|
|
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
|
|
cwd: dir,
|
|
encoding: "utf8",
|
|
timeout: 25000,
|
|
env: {
|
|
...process.env,
|
|
BASH_ENV: resolve(root, "src/test/ci-deploy-harness.sh"),
|
|
TEST_DIR: dir.replaceAll("\\", "/"),
|
|
CMS_DEPLOY_DIR: join(dir, "production").replaceAll("\\", "/"),
|
|
// Deze scenario's testen het in-place pad. Door een pad op te geven
|
|
// dat niet bestaat ziet het script geen blue/green-upstream, ook niet
|
|
// wanneer de test op de productiehost zelf draait.
|
|
CMS_UPSTREAM_FILE: join(dir, "no-such-upstream.conf"),
|
|
CMS_NGINX_SITE: join(dir, "no-such-nginx-site.conf"),
|
|
TEST_SHA: sha,
|
|
SCENARIO: scenario,
|
|
DEPLOY_BRANCH: "main",
|
|
},
|
|
});
|
|
if (result.error) throw result.error;
|
|
return {
|
|
status: result.status,
|
|
output: result.stdout + result.stderr,
|
|
calls: existsSync(join(dir, "calls"))
|
|
? readFileSync(join(dir, "calls"), "utf8")
|
|
: "",
|
|
app: existsSync(join(dir, "epicnext-cms-app"))
|
|
? readFileSync(join(dir, "epicnext-cms-app"), "utf8").trim()
|
|
: null,
|
|
previousRestored: previous ? existsSync(join(dir, previous)) : false,
|
|
backup: existsSync(join(dir, "epicnext-cms-rollback")),
|
|
secondaryBackup: existsSync(join(dir, "epicnext-cms-rollback-secondary")),
|
|
};
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
describe("deployment transaction", () => {
|
|
it("publishes the commit image only after migrations, health and smoke tests", () => {
|
|
const result = simulate("success");
|
|
expect(result.status, result.output).toBe(0);
|
|
expect(result.app).toBe("new");
|
|
expect(result.calls).toContain("node --import tsx e2e/news-real/run.ts");
|
|
expect(result.calls.indexOf("e2e/news-real/run.ts")).toBeLessThan(
|
|
result.calls.indexOf("pnpm db:migrate"),
|
|
);
|
|
expect(result.backup).toBe(false);
|
|
expect(result.calls.indexOf("pnpm db:migrate")).toBeLessThan(
|
|
result.calls.indexOf("docker stop"),
|
|
);
|
|
expect(result.calls.indexOf("verify-deployed-release.mjs")).toBeLessThan(
|
|
result.calls.indexOf("docker tag sha256:new epicnext-cms:latest"),
|
|
);
|
|
expect(result.calls.indexOf("pnpm test:e2e")).toBeLessThan(
|
|
result.calls.indexOf(
|
|
`docker tag sha256:new epicnext-cms:verified-${sha}`,
|
|
),
|
|
);
|
|
expect(result.calls).toContain(
|
|
`docker image rm epicnext-cms:verified-${"c".repeat(40)}`,
|
|
);
|
|
expect(result.calls).not.toContain(
|
|
`docker image rm epicnext-cms:verified-${"b".repeat(40)}`,
|
|
);
|
|
expect(result.calls).not.toContain(
|
|
`docker image rm epicnext-cms:verified-${sha}`,
|
|
);
|
|
expect(result.calls).toContain(
|
|
"docker tag sha256:old epicnext-cms:previous",
|
|
);
|
|
});
|
|
it.each([
|
|
"run-failure",
|
|
"health-failure",
|
|
"smoke-failure",
|
|
"release-failure",
|
|
])(
|
|
"restores the exact previous container after %s",
|
|
(scenario) => {
|
|
const result = simulate(scenario);
|
|
expect(result.status).not.toBe(0);
|
|
expect(result.app).toBe("old");
|
|
expect(result.calls).not.toContain(
|
|
`docker tag sha256:new epicnext-cms:verified-${sha}`,
|
|
);
|
|
expect(result.output).toContain("Rollback verified: sha256:old");
|
|
expect(result.calls).not.toContain(
|
|
"docker tag sha256:new epicnext-cms:latest",
|
|
);
|
|
},
|
|
30000,
|
|
);
|
|
it("restores the legacy compose container on failure", () => {
|
|
const result = simulate("smoke-failure", "epicnext-cms");
|
|
expect(result.status).not.toBe(0);
|
|
expect(result.app).toBeNull();
|
|
expect(result.previousRestored).toBe(true);
|
|
expect(result.calls).toContain("docker start epicnext-cms");
|
|
});
|
|
it.each([
|
|
"lock-failure",
|
|
"remote-failure",
|
|
"build-failure",
|
|
"migration-failure",
|
|
"news-e2e-failure",
|
|
])("leaves the active container untouched after %s", (scenario) => {
|
|
const result = simulate(scenario);
|
|
expect(result.status).not.toBe(0);
|
|
expect(result.app).toBe("old");
|
|
expect(result.calls).not.toContain("docker stop");
|
|
});
|
|
it.each(["stale", "superseded"])(
|
|
"skips a %s commit without touching production",
|
|
(scenario) => {
|
|
const result = simulate(scenario);
|
|
expect(result.status, result.output).toBe(0);
|
|
expect(result.app).toBe("old");
|
|
expect(result.calls).not.toContain("pnpm db:migrate");
|
|
expect(result.calls).not.toContain("docker stop");
|
|
},
|
|
);
|
|
it("reports a failed first deployment without inventing a rollback", () => {
|
|
const result = simulate("smoke-failure", "");
|
|
expect(result.status).not.toBe(0);
|
|
expect(result.output).toContain("No previous container exists");
|
|
expect(result.app).toBeNull();
|
|
});
|
|
});
|
|
|
|
describe("legacy and CI container coexistence", () => {
|
|
it("stops and preserves both before starting the candidate, then removes backups only after verification", () => {
|
|
const r = simulate("success", "epicnext-cms", true);
|
|
expect(r.status, r.output).toBe(0);
|
|
expect(r.calls).toContain("docker stop epicnext-cms-app");
|
|
expect(r.calls).toContain("docker stop epicnext-cms\n");
|
|
expect(r.calls.indexOf("docker stop epicnext-cms\n")).toBeLessThan(
|
|
r.calls.indexOf("docker run"),
|
|
);
|
|
expect(r.calls.indexOf("docker stop epicnext-cms-app")).toBeLessThan(
|
|
r.calls.indexOf("docker run"),
|
|
);
|
|
expect(r.backup).toBe(false);
|
|
expect(r.secondaryBackup).toBe(false);
|
|
expect(r.app).toBe("new");
|
|
});
|
|
it("restores both original containers when candidate release validation fails", () => {
|
|
const r = simulate("release-failure", "epicnext-cms", true);
|
|
expect(r.status, r.output).not.toBe(0);
|
|
expect(r.previousRestored).toBe(true);
|
|
expect(r.app).toBe("old");
|
|
expect(r.secondaryBackup).toBe(false);
|
|
expect(r.calls).toContain("docker start epicnext-cms");
|
|
});
|
|
});
|
|
|
|
// Blue/green: nginx wijst pas naar de nieuwe release nadat die gezond is én de
|
|
// e2e-test heeft gewonnen. De hele winst zit in de volgorde, dus daar asserten
|
|
// we dan ook op.
|
|
function simulateBlueGreen(scenario: string, livePort = 3002) {
|
|
const dir = mkdtempSync(join(tmpdir(), "cms-deploy-bg-"));
|
|
try {
|
|
mkdirSync(join(dir, "production"));
|
|
// Een werkende deploymap heeft een DATABASE_URL; die gebruikt
|
|
// scripts/ci-deploy.sh voor de migraties op de host.
|
|
writeFileSync(
|
|
join(dir, "production", ".env"),
|
|
'HOTEL_NAME="Test Hotel"\nDATABASE_URL="mysql://test:[email protected]:3306/test"\n',
|
|
);
|
|
// De live release draait op `livePort`; nginx wijst ernaar.
|
|
writeFileSync(
|
|
join(dir, "upstream.conf"),
|
|
`server 127.0.0.1:${livePort} max_fails=2 fail_timeout=10s;\n`,
|
|
);
|
|
writeFileSync(join(dir, "nginx-site.conf"), "upstream cms_app { }\n");
|
|
// Het live container-bestand, zodat `docker inspect` hem als draaiend ziet.
|
|
writeFileSync(join(dir, "epicnext-cms-app"), "old\n");
|
|
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
|
|
cwd: dir,
|
|
encoding: "utf8",
|
|
timeout: 25000,
|
|
env: {
|
|
...process.env,
|
|
BASH_ENV: resolve(root, "src/test/ci-deploy-harness.sh"),
|
|
TEST_DIR: dir.replaceAll("\\", "/"),
|
|
CMS_DEPLOY_DIR: join(dir, "production").replaceAll("\\", "/"),
|
|
CMS_UPSTREAM_FILE: join(dir, "upstream.conf"),
|
|
CMS_NGINX_SITE: join(dir, "nginx-site.conf"),
|
|
TEST_SHA: sha,
|
|
SCENARIO: scenario,
|
|
DEPLOY_BRANCH: "main",
|
|
},
|
|
});
|
|
if (result.error) throw result.error;
|
|
return {
|
|
status: result.status,
|
|
output: result.stdout + result.stderr,
|
|
calls: existsSync(join(dir, "calls"))
|
|
? readFileSync(join(dir, "calls"), "utf8")
|
|
: "",
|
|
upstream: existsSync(join(dir, "upstream.conf"))
|
|
? readFileSync(join(dir, "upstream.conf"), "utf8")
|
|
: "",
|
|
green: existsSync(join(dir, "epicnext-cms-green"))
|
|
? readFileSync(join(dir, "epicnext-cms-green"), "utf8").trim()
|
|
: null,
|
|
app: existsSync(join(dir, "epicnext-cms-app"))
|
|
? readFileSync(join(dir, "epicnext-cms-app"), "utf8").trim()
|
|
: null,
|
|
};
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
describe("blue/green cutover", () => {
|
|
it("starts the candidate on the idle port and keeps the live one serving", () => {
|
|
const r = simulateBlueGreen("success");
|
|
expect(r.status, r.output).toBe(0);
|
|
// De kandidaat draait naast de live release, niet ervoor.
|
|
expect(r.calls).toContain("docker run -d --name epicnext-cms-green");
|
|
expect(r.calls).toContain("PORT=3003");
|
|
expect(r.calls.indexOf("docker run")).toBeLessThan(
|
|
r.calls.indexOf("docker stop epicnext-cms-app"),
|
|
);
|
|
// De release-hash en de e2e-test draaien tegen de kandidaat, vóór de swap.
|
|
expect(r.calls).toContain("http://127.0.0.1:3003/api/health");
|
|
expect(r.calls).toContain("PLAYWRIGHT_BASE_URL=http://127.0.0.1:3003");
|
|
expect(r.calls.indexOf("pnpm test:e2e")).toBeLessThan(
|
|
r.calls.indexOf("nginx -s reload"),
|
|
);
|
|
});
|
|
|
|
it("switches nginx over before retiring the live container", () => {
|
|
const r = simulateBlueGreen("success");
|
|
expect(r.status, r.output).toBe(0);
|
|
// nginx test vóór de reload, en de swap gaat vóór het stoppen: anders
|
|
// zou er een moment zijn waarop niets draait.
|
|
expect(r.calls).toContain("nginx -t");
|
|
expect(r.calls.indexOf("nginx -t")).toBeLessThan(
|
|
r.calls.indexOf("nginx -s reload"),
|
|
);
|
|
expect(r.calls.indexOf("nginx -s reload")).toBeLessThan(
|
|
r.calls.indexOf("docker stop epicnext-cms-app"),
|
|
);
|
|
expect(r.upstream).toContain("127.0.0.1:3003");
|
|
expect(r.upstream).not.toContain("127.0.0.1:3002");
|
|
});
|
|
|
|
it("alternates slots on the next release", () => {
|
|
const r = simulateBlueGreen("success", 3003);
|
|
expect(r.status, r.output).toBe(0);
|
|
// Live op 3003 → de kandidaat gaat naar 3002.
|
|
expect(r.calls).toContain("docker run -d --name epicnext-cms-app");
|
|
expect(r.upstream).toContain("127.0.0.1:3002");
|
|
expect(r.upstream).not.toContain("127.0.0.1:3003");
|
|
});
|
|
|
|
// Regressie: een poort die al in gebruik is liet `docker run` stilletjes op
|
|
// EADDRINUSE sterven. De health-probe beantwoordde daarna vanaf de
|
|
// reeds draaiende container op diezelfde poort, waardoor de
|
|
// release-vergelijking 30 keer op een verkeerde release faalde in plaats
|
|
// van op de echte oorzaak te wijzen.
|
|
it("refuses to start the candidate on a port that is already in use", () => {
|
|
const r = simulateBlueGreen("port-taken");
|
|
expect(r.status, r.output).not.toBe(0);
|
|
expect(r.output).toContain("already in use");
|
|
// Er is geen kandidaat gestart, dus er is ook niets om te verwijderen.
|
|
expect(r.calls).not.toContain("docker run");
|
|
// De live release draait ongestoord door en nginx wijst nog steeds
|
|
// naar de oude poort: geen halve cutover.
|
|
expect(r.upstream).toContain("127.0.0.1:3002");
|
|
expect(r.upstream).not.toContain("127.0.0.1:3003");
|
|
});
|
|
|
|
it.each([
|
|
"run-failure",
|
|
"health-failure",
|
|
"smoke-failure",
|
|
"release-failure",
|
|
])(
|
|
"never stops the live release when the candidate fails on %s",
|
|
(scenario) => {
|
|
const r = simulateBlueGreen(scenario);
|
|
expect(r.status, r.output).not.toBe(0);
|
|
// Dit is de hele belofte: een mislukte release kost tijd, geen downtime.
|
|
expect(r.calls).not.toContain("docker stop epicnext-cms-app");
|
|
expect(r.calls).not.toContain("nginx -s reload");
|
|
expect(r.upstream).toContain("127.0.0.1:3002");
|
|
expect(r.green).toBeNull();
|
|
expect(r.output).toContain("the live release was never stopped");
|
|
},
|
|
30000,
|
|
);
|
|
|
|
it("leaves the live release alone when migrations fail", () => {
|
|
// Migraties draaien vóórdat de blue/green-detectie, dus er is op dat
|
|
// moment simpelweg nog niets om terug te draaien.
|
|
const r = simulateBlueGreen("migration-failure");
|
|
expect(r.status, r.output).not.toBe(0);
|
|
expect(r.calls).not.toContain("docker stop");
|
|
expect(r.calls).not.toContain("docker run");
|
|
expect(r.calls).not.toContain("nginx -s reload");
|
|
expect(r.upstream).toContain("127.0.0.1:3002");
|
|
expect(r.app).toBe("old");
|
|
});
|
|
|
|
it("keeps the live release serving when nginx rejects the new upstream", () => {
|
|
const r = simulateBlueGreen("nginx-reject");
|
|
expect(r.status, r.output).not.toBe(0);
|
|
expect(r.calls).not.toContain("docker stop epicnext-cms-app");
|
|
expect(r.calls).not.toContain("nginx -s reload");
|
|
// De upstream is teruggezet op wat er stond.
|
|
expect(r.upstream).toContain("127.0.0.1:3002");
|
|
expect(r.output).toContain("restoring the previous one");
|
|
});
|
|
});
|