retention + cleanup: channel retention policy, store GC, deletion audit (P5.3)
Subplan 02 Slice 4. Channel gains retention_keep (0 = keep everything; N = keep the newest N published releases of the channel's lineage), set via the new `dist channel set --retention-keep`. The new `dist store cleanup` prunes lineage-expired releases — never one any channel points at, so cross-promoted releases survive — drops their artifact rows, GCs objects/ files no surviving artifact references, and sweeps stale staging/ leftovers; every deletion writes an audit event. The pruned model is saved before any unlink, so a crash leaves orphan blobs (next run catches them), never dangling references. repo.publish no longer replaces an existing channel row wholesale: only the pointer moves, so policy/rollout/retention survive every publish (previously each publish silently reset them to defaults). std.fs has no directory listing, so cleanup.sx carries a local opendir/readdir/closedir shim, like publish.sx's time(2) shim. dist.db channels gains the retention_keep column (idempotent ALTER for pre-retention stores); db.json import treats it as optional. tests/retention_cleanup.sx pins the whole scenario; the repo.publish assertion fails on the pre-fix code. make test 23/23 green.
This commit is contained in:
agra
3
.gitignore
vendored
3
.gitignore
vendored
@@ -3,6 +3,9 @@
|
||||
# build artifacts from `make build`
|
||||
build/
|
||||
|
||||
# a root-level `sx build -o dist` convenience binary
|
||||
/dist
|
||||
|
||||
# scratch store roots / fixtures created by tests (never /tmp)
|
||||
.sx-tmp/
|
||||
|
||||
|
||||
94
src/dist.sx
94
src/dist.sx
@@ -12,6 +12,8 @@
|
||||
// dist token create mint a scoped CI token, secret shown once (P4.3) — see token/ops.sx
|
||||
// dist token list tokens with lifecycle status, never the secret (P4.3)
|
||||
// dist token revoke revoke a token by id (P4.3)
|
||||
// dist channel set set a channel's retention policy (P5.3) — see release/ops.sx
|
||||
// dist store cleanup prune retention-expired releases + GC the store (P5.3) — see store/cleanup.sx
|
||||
//
|
||||
// EXIT-CODE CONTRACT (sysexits, via std.cli): success ends with
|
||||
// `exit_ok()` (EX_OK = 0); a no-command / unknown-or-missing
|
||||
@@ -37,6 +39,7 @@ rem :: #import "publish/remote.sx";
|
||||
ops :: #import "release/ops.sx";
|
||||
tops :: #import "token/ops.sx";
|
||||
aops :: #import "app/ops.sx";
|
||||
clup :: #import "store/cleanup.sx";
|
||||
srv :: #import "server/distd.sx";
|
||||
|
||||
// Direct stderr writer (fd 2), so human help/usage/progress never lands on
|
||||
@@ -52,7 +55,7 @@ emit_human :: (s: string, json_mode: bool) {
|
||||
if json_mode { eputs(s); } else { out(s); }
|
||||
}
|
||||
|
||||
HELP :: "dist — application distribution CLI\n\nUsage:\n dist <group> <command> [flags]\n\nGroups & commands:\n ci\n ci publish publish a release from CI\n --manifest <path> publish manifest (dist.json) to read\n --local-store <dir> publish into a local store directory, OR:\n --server <url> publish against a running distd (http://<ipv4-or-localhost>:<port>)\n --token <secret> bearer token for --server (mint with: dist token create)\n release\n release promote point a channel at a release\n --app <slug> app the channel belongs to\n --channel <name> channel to move\n --release <id> release id to promote\n --local-store <dir> local artifact store + dist.db directory\n release rollback move a channel back to its previous release\n --app <slug> app the channel belongs to\n --channel <name> channel to roll back\n --local-store <dir> local artifact store + dist.db directory\n server\n server run serve the store over HTTP (0.0.0.0)\n --local-store <dir> local artifact store + dist.db directory\n --port <n> TCP port (default 8787)\n GET (public): / (HTML index), /healthz, /api/apps, /api/apps/<slug>, /download/<sha256>\n POST (Bearer token, publish scope): /api/upload, /api/apps/<slug>/releases,\n /api/apps/<slug>/channels/<name>/promote, /api/apps/<slug>/channels/<name>/rollback\n app\n app set edit an existing app's display name / iOS install policy\n --app <slug> app to edit (apps are created by publish)\n --local-store <dir> local artifact store + dist.db directory\n --display-name <s> new display name\n --ios-mode <m> artifact_only | testflight | enterprise\n --testflight-url <u> TestFlight link (required for testflight mode)\n --ios-bundle-id <id> iOS bundle identifier (required for enterprise mode)\n token\n token create mint a scoped automation token (secret shown ONCE)\n --name <name> token name, [a-z0-9._-]\n --local-store <dir> local artifact store + dist.db directory\n --scope <words> space-separated scopes: publish read (default: publish)\n --app <slug> restrict to one app (default: any)\n --channel <name> restrict to one channel (default: any)\n --expires-in <secs> lifetime in seconds (default: never expires)\n token list tokens with lifecycle status (never the secret)\n --local-store <dir> local artifact store + dist.db directory\n token revoke revoke a token by id\n --id <token-id> token to revoke\n --local-store <dir> local artifact store + dist.db directory\n\nGlobal flags:\n --json emit machine-readable JSON on stdout; human text to stderr\n -h, --help show this help and exit\n\nExit codes:\n 0 success\n 1 command failed (publish/promote/rollback/token op aborted or server could not bind)\n 64 usage error (no command, or an unknown/missing command or flag)\n";
|
||||
HELP :: "dist — application distribution CLI\n\nUsage:\n dist <group> <command> [flags]\n\nGroups & commands:\n ci\n ci publish publish a release from CI\n --manifest <path> publish manifest (dist.json) to read\n --local-store <dir> publish into a local store directory, OR:\n --server <url> publish against a running distd (http://<ipv4-or-localhost>:<port>)\n --token <secret> bearer token for --server (mint with: dist token create)\n release\n release promote point a channel at a release\n --app <slug> app the channel belongs to\n --channel <name> channel to move\n --release <id> release id to promote\n --local-store <dir> local artifact store + dist.db directory\n release rollback move a channel back to its previous release\n --app <slug> app the channel belongs to\n --channel <name> channel to roll back\n --local-store <dir> local artifact store + dist.db directory\n server\n server run serve the store over HTTP (0.0.0.0)\n --local-store <dir> local artifact store + dist.db directory\n --port <n> TCP port (default 8787)\n GET (public): / (HTML index), /healthz, /api/apps, /api/apps/<slug>, /download/<sha256>\n POST (Bearer token, publish scope): /api/upload, /api/apps/<slug>/releases,\n /api/apps/<slug>/channels/<name>/promote, /api/apps/<slug>/channels/<name>/rollback\n channel\n channel set set an existing channel's retention policy\n --app <slug> app the channel belongs to\n --channel <name> channel to edit\n --retention-keep <n> keep the newest n published releases (0 = keep everything)\n --local-store <dir> local artifact store + dist.db directory\n store\n store cleanup prune retention-expired releases and GC the store\n --local-store <dir> local artifact store + dist.db directory\n deletes releases beyond each channel's retention (never one a channel\n points at), objects no artifact references, and stale staging files;\n every deletion writes an audit event\n app\n app set edit an existing app's display name / iOS install policy\n --app <slug> app to edit (apps are created by publish)\n --local-store <dir> local artifact store + dist.db directory\n --display-name <s> new display name\n --ios-mode <m> artifact_only | testflight | enterprise\n --testflight-url <u> TestFlight link (required for testflight mode)\n --ios-bundle-id <id> iOS bundle identifier (required for enterprise mode)\n token\n token create mint a scoped automation token (secret shown ONCE)\n --name <name> token name, [a-z0-9._-]\n --local-store <dir> local artifact store + dist.db directory\n --scope <words> space-separated scopes: publish read (default: publish)\n --app <slug> restrict to one app (default: any)\n --channel <name> restrict to one channel (default: any)\n --expires-in <secs> lifetime in seconds (default: never expires)\n token list tokens with lifecycle status (never the secret)\n --local-store <dir> local artifact store + dist.db directory\n token revoke revoke a token by id\n --id <token-id> token to revoke\n --local-store <dir> local artifact store + dist.db directory\n\nGlobal flags:\n --json emit machine-readable JSON on stdout; human text to stderr\n -h, --help show this help and exit\n\nExit codes:\n 0 success\n 1 command failed (publish/promote/rollback/token op aborted or server could not bind)\n 64 usage error (no command, or an unknown/missing command or flag)\n";
|
||||
|
||||
// True if `name` appears as a token in `args`.
|
||||
has_flag :: (args: []string, name: string) -> bool {
|
||||
@@ -360,6 +363,82 @@ handle_app_set :: (p: *Parsed, json_mode: bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// `dist channel set` — set an existing channel's retention policy over the
|
||||
// persisted store (P5.3). Rendering contract as above.
|
||||
handle_channel_set :: (p: *Parsed, json_mode: bool) {
|
||||
keepq := parse_keep(p.value_of("retention-keep"));
|
||||
if keepq == null {
|
||||
eputs(concat(concat("dist: --retention-keep must be 0 (keep everything) or a positive release count, got: ", p.value_of("retention-keep")), "\n"));
|
||||
exit_usage();
|
||||
}
|
||||
|
||||
fail : jout.CliFailure = .{};
|
||||
o, e := ops.run_channel_set(p.value_of("local-store"), p.value_of("app"),
|
||||
p.value_of("channel"), keepq!, @fail);
|
||||
if e {
|
||||
report_failure("channel set", fail, json_mode);
|
||||
}
|
||||
if !e {
|
||||
if !json_mode {
|
||||
out(ops.channel_set_human(@o));
|
||||
return;
|
||||
}
|
||||
eputs("dist: channel set ok\n");
|
||||
raw : [4096]u8 = ---;
|
||||
werr := false;
|
||||
n := ops.write_channel_set_json(@o, string.{ ptr = @raw[0], len = 4096 }) catch { werr = true; 0 };
|
||||
if werr {
|
||||
eputs("dist: internal error: JSON serialization failed\n");
|
||||
exit_command_failed();
|
||||
}
|
||||
out(string.{ ptr = @raw[0], len = n });
|
||||
out("\n");
|
||||
}
|
||||
}
|
||||
|
||||
// `dist store cleanup` — prune retention-expired releases, GC unreferenced
|
||||
// objects, and sweep stale staging files (P5.3). Rendering contract as
|
||||
// above.
|
||||
handle_store_cleanup :: (p: *Parsed, json_mode: bool) {
|
||||
fail : jout.CliFailure = .{};
|
||||
o, e := clup.run_cleanup(p.value_of("local-store"), @fail);
|
||||
if e {
|
||||
report_failure("store cleanup", fail, json_mode);
|
||||
}
|
||||
if !e {
|
||||
if !json_mode {
|
||||
out(clup.cleanup_human(@o));
|
||||
return;
|
||||
}
|
||||
eputs("dist: store cleanup ok\n");
|
||||
raw : [16384]u8 = ---;
|
||||
werr := false;
|
||||
n := clup.write_cleanup_json(@o, string.{ ptr = @raw[0], len = 16384 }) catch { werr = true; 0 };
|
||||
if werr {
|
||||
eputs("dist: internal error: JSON serialization failed\n");
|
||||
exit_command_failed();
|
||||
}
|
||||
out(string.{ ptr = @raw[0], len = n });
|
||||
out("\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Non-negative decimal release count (a retention policy; 0 = keep
|
||||
// everything); anything else (empty, non-digits, absurd length) is null —
|
||||
// a usage error at the call site.
|
||||
parse_keep :: (s: string) -> ?i64 {
|
||||
if s.len == 0 or s.len > 12 { return null; }
|
||||
v : i64 = 0;
|
||||
i := 0;
|
||||
while i < s.len {
|
||||
c := s[i];
|
||||
if c < 48 or c > 57 { return null; } // '0'..'9'
|
||||
v = v * 10 + (c - 48);
|
||||
i += 1;
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
// Positive decimal seconds (a token lifetime); anything else (empty,
|
||||
// non-digits, zero, absurd length) is null — a usage error at the call
|
||||
// site.
|
||||
@@ -404,6 +483,8 @@ dispatch :: (p: *Parsed, json_mode: bool) {
|
||||
if p.group == "token" and p.command == "list" { handle_token_list(p, json_mode); return; }
|
||||
if p.group == "token" and p.command == "revoke" { handle_token_revoke(p, json_mode); return; }
|
||||
if p.group == "app" and p.command == "set" { handle_app_set(p, json_mode); return; }
|
||||
if p.group == "channel" and p.command == "set" { handle_channel_set(p, json_mode); return; }
|
||||
if p.group == "store" and p.command == "cleanup" { handle_store_cleanup(p, json_mode); return; }
|
||||
eputs("dist: internal error: unrouted command\n");
|
||||
exit_usage();
|
||||
}
|
||||
@@ -470,6 +551,15 @@ main :: () -> ! {
|
||||
FlagSpec.{ name = "id", takes_value = true, required = true },
|
||||
FlagSpec.{ name = "local-store", takes_value = true, required = true },
|
||||
];
|
||||
channel_set_flags : []FlagSpec = .[
|
||||
FlagSpec.{ name = "app", takes_value = true, required = true },
|
||||
FlagSpec.{ name = "channel", takes_value = true, required = true },
|
||||
FlagSpec.{ name = "retention-keep", takes_value = true, required = true },
|
||||
FlagSpec.{ name = "local-store", takes_value = true, required = true },
|
||||
];
|
||||
store_cleanup_flags : []FlagSpec = .[
|
||||
FlagSpec.{ name = "local-store", takes_value = true, required = true },
|
||||
];
|
||||
app_set_flags : []FlagSpec = .[
|
||||
FlagSpec.{ name = "app", takes_value = true, required = true },
|
||||
FlagSpec.{ name = "local-store", takes_value = true, required = true },
|
||||
@@ -487,6 +577,8 @@ main :: () -> ! {
|
||||
Command.{ group = "token", command = "list", flags = token_list_flags },
|
||||
Command.{ group = "token", command = "revoke", flags = token_revoke_flags },
|
||||
Command.{ group = "app", command = "set", flags = app_set_flags },
|
||||
Command.{ group = "channel", command = "set", flags = channel_set_flags },
|
||||
Command.{ group = "store", command = "cleanup", flags = store_cleanup_flags },
|
||||
];
|
||||
|
||||
diag : Diag = .{};
|
||||
|
||||
@@ -9,11 +9,16 @@ RolloutPolicy :: enum u8 {
|
||||
|
||||
// A named publishing track for an app (e.g. "stable", "beta"), pointing at
|
||||
// the release currently serving traffic. `rollout_percent` is meaningful
|
||||
// only under the `percentage` policy.
|
||||
// only under the `percentage` policy. `retention_keep` is the channel's
|
||||
// retention policy: 0 keeps every release forever; N > 0 marks all but the
|
||||
// newest N published releases of this channel's lineage as deletable by
|
||||
// `dist store cleanup` (a release any channel currently points at is never
|
||||
// deleted, regardless of retention).
|
||||
Channel :: struct {
|
||||
app_id: string;
|
||||
name: string;
|
||||
current_release_id: string;
|
||||
policy: RolloutPolicy = .manual;
|
||||
rollout_percent: i64 = 100;
|
||||
retention_keep: i64 = 0;
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ ValidationErr :: error {
|
||||
BadChannelName, // channel name empty or outside [a-z0-9-]
|
||||
UnknownPlatform, // platform id did not name a Platform variant
|
||||
BadRollout, // rollout_percent outside 0..100
|
||||
BadRetention, // retention_keep negative
|
||||
BadSize, // artifact size_bytes was not positive
|
||||
BadDigest, // sha256 was not exactly 64 lowercase-hex chars
|
||||
BadTokenName, // token name empty or outside [a-z0-9._-]
|
||||
@@ -192,6 +193,7 @@ validate_channel :: (c: Channel) -> !ValidationErr {
|
||||
if c.app_id.len == 0 { raise error.MissingField; }
|
||||
try validate_channel_name(c.name);
|
||||
if c.rollout_percent < 0 or c.rollout_percent > 100 { raise error.BadRollout; }
|
||||
if c.retention_keep < 0 { raise error.BadRetention; }
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -72,6 +72,12 @@ RollbackOutcome :: struct {
|
||||
to_version: string;
|
||||
}
|
||||
|
||||
ChannelSetOutcome :: struct {
|
||||
app_id: string;
|
||||
channel: string;
|
||||
retention_keep: i64;
|
||||
}
|
||||
|
||||
// ── shared steps ──────────────────────────────────────────────────────
|
||||
|
||||
// Load the persisted model, or fail with `store.load` when the store has
|
||||
@@ -240,6 +246,51 @@ run_rollback :: (store_dir: string, app_slug: string, channel_name: string, fail
|
||||
};
|
||||
}
|
||||
|
||||
// ── channel set ───────────────────────────────────────────────────────
|
||||
|
||||
// `dist channel set` — edit an existing channel's retention policy
|
||||
// (subplan 02 Slice 4). Channels are created by publish/promote; setting
|
||||
// retention on a channel that doesn't exist yet is rejected rather than
|
||||
// creating a pointer-less channel. `retention_keep` < 0 fails domain
|
||||
// validation; 0 = keep everything.
|
||||
run_channel_set :: (store_dir: string, app_slug: string, channel_name: string, retention_keep: i64, fail_out: *jout.CliFailure) -> (ChannelSetOutcome, !OpError) {
|
||||
repo := try op_load_repo(store_dir, fail_out);
|
||||
app := try op_find_app(@repo, app_slug, "channel.unknown_app", fail_out);
|
||||
|
||||
cq := repo.get_channel(app.id, channel_name);
|
||||
if cq == null {
|
||||
fail_out.code = "channel.unknown";
|
||||
fail_out.message = concat("the app has no channel with that name (publish/promote creates channels): ", channel_name);
|
||||
raise error.NotFound;
|
||||
}
|
||||
chan := cq!;
|
||||
chan.retention_keep = retention_keep;
|
||||
|
||||
cverr := false;
|
||||
validate_channel(chan) catch { cverr = true; };
|
||||
if cverr {
|
||||
fail_out.code = "channel.bad_retention";
|
||||
fail_out.message = "--retention-keep must be 0 (keep everything) or a positive release count";
|
||||
raise error.Invalid;
|
||||
}
|
||||
repo.update_channel(chan);
|
||||
|
||||
now := pl.now_secs();
|
||||
repo.create_audit_event(AuditEvent.{
|
||||
id = concat(concat(concat("evt-cli-channel-set-", app.id), "-"), channel_name),
|
||||
actor = "cli", action = "channel.update", target_type = "channel",
|
||||
target_id = channel_name,
|
||||
metadata = concat("retention_keep=", int_to_string(retention_keep)),
|
||||
created_at = now,
|
||||
});
|
||||
|
||||
try op_save(@repo, store_dir, fail_out);
|
||||
|
||||
return ChannelSetOutcome.{
|
||||
app_id = app.id, channel = channel_name, retention_keep = retention_keep,
|
||||
};
|
||||
}
|
||||
|
||||
// ── rendering ─────────────────────────────────────────────────────────
|
||||
|
||||
// `{"status":"promoted","app_id":...,"channel":...,"release":{"id":...,
|
||||
@@ -279,6 +330,32 @@ write_rollback_json :: (o: *RollbackOutcome, dst: []u8) -> (i64, !JsonError) {
|
||||
return n;
|
||||
}
|
||||
|
||||
// `{"status":"updated","channel":{"app_id":...,"name":...,
|
||||
// "retention_keep":N}}`.
|
||||
write_channel_set_json :: (o: *ChannelSetOutcome, dst: []u8) -> (i64, !JsonError) {
|
||||
gpa := GPA.init();
|
||||
root : Object = .{};
|
||||
root.put("status", .str("updated"), xx gpa);
|
||||
co : Object = .{};
|
||||
co.put("app_id", .str(o.app_id), xx gpa);
|
||||
co.put("name", .str(o.channel), xx gpa);
|
||||
co.put("retention_keep", .int_(o.retention_keep), xx gpa);
|
||||
root.put("channel", .object(co), xx gpa);
|
||||
rootv : Value = .object(root);
|
||||
n := try write_to_buffer(rootv, dst);
|
||||
return n;
|
||||
}
|
||||
|
||||
channel_set_human :: (o: *ChannelSetOutcome) -> string {
|
||||
s := concat("updated channel ", o.channel);
|
||||
if o.retention_keep == 0 {
|
||||
s = concat(s, ": retention keep everything");
|
||||
} else {
|
||||
s = concat(s, concat(": retention keep last ", int_to_string(o.retention_keep)));
|
||||
}
|
||||
return concat(s, "\n");
|
||||
}
|
||||
|
||||
promote_human :: (o: *PromoteOutcome) -> string {
|
||||
s := concat("promoted ", o.release_id);
|
||||
s = concat(s, concat(" (", concat(o.version, ")")));
|
||||
|
||||
@@ -137,7 +137,10 @@ db_ensure_schema :: (conn: *Sqlite) -> bool {
|
||||
if ok { conn.exec("CREATE INDEX IF NOT EXISTS idx_releases_app_id ON releases (app_id)") catch { ok = false; }; }
|
||||
if ok { conn.exec("CREATE TABLE IF NOT EXISTS artifacts (id TEXT NOT NULL, app_id TEXT NOT NULL, release_id TEXT NOT NULL, platform TEXT NOT NULL, filename TEXT NOT NULL, content_type TEXT NOT NULL, size_bytes INTEGER NOT NULL, sha256 TEXT NOT NULL, storage_key TEXT NOT NULL, metadata TEXT NOT NULL, validation_status TEXT NOT NULL)") catch { ok = false; }; }
|
||||
if ok { conn.exec("CREATE INDEX IF NOT EXISTS idx_artifacts_sha256 ON artifacts (sha256)") catch { ok = false; }; }
|
||||
if ok { conn.exec("CREATE TABLE IF NOT EXISTS channels (app_id TEXT NOT NULL, name TEXT NOT NULL, current_release_id TEXT NOT NULL, policy TEXT NOT NULL, rollout_percent INTEGER NOT NULL, UNIQUE (app_id, name))") catch { ok = false; }; }
|
||||
if ok { conn.exec("CREATE TABLE IF NOT EXISTS channels (app_id TEXT NOT NULL, name TEXT NOT NULL, current_release_id TEXT NOT NULL, policy TEXT NOT NULL, rollout_percent INTEGER NOT NULL, retention_keep INTEGER NOT NULL DEFAULT 0, UNIQUE (app_id, name))") catch { ok = false; }; }
|
||||
// Pre-retention databases lack the column; on those the ALTER adds it,
|
||||
// everywhere else it fails with "duplicate column" and is ignored.
|
||||
if ok { conn.exec("ALTER TABLE channels ADD COLUMN retention_keep INTEGER NOT NULL DEFAULT 0") catch {}; }
|
||||
if ok { conn.exec("CREATE TABLE IF NOT EXISTS tokens (id TEXT PRIMARY KEY, name TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, scopes TEXT NOT NULL, app_slug TEXT NOT NULL, channel TEXT NOT NULL, created_at INTEGER NOT NULL, expires_at INTEGER NOT NULL, last_used_at INTEGER NOT NULL, revoked_at INTEGER NOT NULL)") catch { ok = false; }; }
|
||||
if ok { conn.exec("CREATE TABLE IF NOT EXISTS audit_events (id TEXT NOT NULL, actor TEXT NOT NULL, action TEXT NOT NULL, target_type TEXT NOT NULL, target_id TEXT NOT NULL, metadata TEXT NOT NULL, created_at INTEGER NOT NULL)") catch { ok = false; }; }
|
||||
return ok;
|
||||
@@ -259,7 +262,7 @@ db_write_artifacts :: (repo: *Repo, conn: *Sqlite) -> bool {
|
||||
}
|
||||
|
||||
db_write_channels :: (repo: *Repo, conn: *Sqlite) -> bool {
|
||||
st, pe := conn.prepare("INSERT INTO channels (app_id, name, current_release_id, policy, rollout_percent) VALUES (?1, ?2, ?3, ?4, ?5)");
|
||||
st, pe := conn.prepare("INSERT INTO channels (app_id, name, current_release_id, policy, rollout_percent, retention_keep) VALUES (?1, ?2, ?3, ?4, ?5, ?6)");
|
||||
if pe { return false; }
|
||||
ok := true;
|
||||
i := 0;
|
||||
@@ -270,6 +273,7 @@ db_write_channels :: (repo: *Repo, conn: *Sqlite) -> bool {
|
||||
st.bind_text(3, c.current_release_id) catch { ok = false; };
|
||||
st.bind_text(4, policy_str(c.policy)) catch { ok = false; };
|
||||
st.bind_int64(5, c.rollout_percent) catch { ok = false; };
|
||||
st.bind_int64(6, c.retention_keep) catch { ok = false; };
|
||||
if ok {
|
||||
rc, se := st.step();
|
||||
if se { ok = false; }
|
||||
@@ -473,7 +477,7 @@ db_read_artifacts :: (repo: *Repo, conn: *Sqlite) -> !LoadErr {
|
||||
}
|
||||
|
||||
db_read_channels :: (repo: *Repo, conn: *Sqlite) -> !LoadErr {
|
||||
st, pe := conn.prepare("SELECT app_id, name, current_release_id, policy, rollout_percent FROM channels ORDER BY rowid");
|
||||
st, pe := conn.prepare("SELECT app_id, name, current_release_id, policy, rollout_percent, retention_keep FROM channels ORDER BY rowid");
|
||||
if pe { raise error.Io; }
|
||||
io_bad := false;
|
||||
shape_bad := false;
|
||||
@@ -489,6 +493,7 @@ db_read_channels :: (repo: *Repo, conn: *Sqlite) -> !LoadErr {
|
||||
if ple { shape_bad = true; break; }
|
||||
c.policy = pol;
|
||||
c.rollout_percent = st.column_int64(4);
|
||||
c.retention_keep = st.column_int64(5);
|
||||
repo.create_channel(c);
|
||||
}
|
||||
st.finalize();
|
||||
@@ -646,6 +651,7 @@ channel_to_json :: (c: Channel, alloc: Allocator) -> Value {
|
||||
o.put("current_release_id", .str(c.current_release_id), alloc);
|
||||
o.put("policy", .str(policy_str(c.policy)), alloc);
|
||||
o.put("rollout_percent", .int_(c.rollout_percent), alloc);
|
||||
o.put("retention_keep", .int_(c.retention_keep), alloc);
|
||||
return .object(o);
|
||||
}
|
||||
|
||||
@@ -790,6 +796,14 @@ channel_from_json :: (o: Object, alloc: Allocator) -> (Channel, !LoadErr) {
|
||||
c.current_release_id = try req_str(o, "current_release_id", alloc);
|
||||
c.policy = try parse_policy(try req_str_view(o, "policy"));
|
||||
c.rollout_percent = try req_int(o, "rollout_percent");
|
||||
// OPTIONAL member (absent in db.json files from before retention
|
||||
// landed): default 0 = keep everything. A present member is strict.
|
||||
rkq := db_obj_find(o, "retention_keep");
|
||||
if rkq != null {
|
||||
rkv := rkq!;
|
||||
if rkv != .int_ { raise error.BadShape; }
|
||||
c.retention_keep = rkv.int_;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
@@ -292,7 +292,11 @@ Repo :: struct {
|
||||
}
|
||||
}
|
||||
if !failed {
|
||||
// An existing channel only has its pointer moved: policy,
|
||||
// rollout, and retention configured on it survive every publish.
|
||||
// The caller's `chan` fields shape the channel only on creation.
|
||||
c := chan;
|
||||
if cidx >= 0 { c = self.channels.items[cidx]; }
|
||||
c.current_release_id = release.id;
|
||||
validate_channel(c) catch { failed = true; };
|
||||
if !failed {
|
||||
|
||||
314
src/store/cleanup.sx
Normal file
314
src/store/cleanup.sx
Normal file
@@ -0,0 +1,314 @@
|
||||
// =====================================================================
|
||||
// cleanup.sx — `dist store cleanup`: retention pruning + store garbage
|
||||
// collection over the persisted store (subplan 02, Slice 4).
|
||||
//
|
||||
// One pass, four deletions, all audited:
|
||||
//
|
||||
// 1. RELEASES beyond retention. For every channel with
|
||||
// `retention_keep` = N > 0, the channel's lineage (the app's
|
||||
// PUBLISHED releases targeting that channel, in publish order)
|
||||
// keeps its newest N entries; older ones are deleted — UNLESS a
|
||||
// release is currently pointed at by ANY channel (cross-promotion
|
||||
// means e.g. stable may point into beta's lineage), in which case
|
||||
// it survives and is reported in `kept_pointed_releases`.
|
||||
// 2. ARTIFACT rows of the deleted releases.
|
||||
// 3. OBJECT files in `<store>/objects/` that no surviving artifact
|
||||
// references (covers both blobs freed by 1/2 and blobs orphaned by
|
||||
// an aborted upload that never reached a release).
|
||||
// 4. EVERYTHING in `<store>/staging/`. A completed put always renames
|
||||
// or deletes its staging file, so anything still there is a
|
||||
// leftover from a crashed/aborted publish.
|
||||
//
|
||||
// ORDERING: the pruned model (with one audit event per deletion) is
|
||||
// saved to dist.db BEFORE any file is unlinked, so a crash mid-cleanup
|
||||
// can leave an unreferenced blob (the next run catches it) but never a
|
||||
// model row pointing at deleted bytes. An unlink that fails after the
|
||||
// save is reported in `unlink_failed` (exit stays 0 — the model is
|
||||
// consistent; the orphan is re-detected next run).
|
||||
//
|
||||
// OFFLINE OP: like every load-modify-save CLI command, cleanup must not
|
||||
// run concurrently with a writing distd — the whole-model save would
|
||||
// drop a publish that landed in between, and the staging sweep would
|
||||
// eat an in-flight upload's temp file.
|
||||
// =====================================================================
|
||||
|
||||
#import "modules/std.sx";
|
||||
#import "modules/std/json.sx";
|
||||
fs :: #import "modules/std/fs.sx";
|
||||
#import "../domain/platform.sx";
|
||||
#import "../domain/app.sx";
|
||||
#import "../domain/release.sx";
|
||||
#import "../domain/artifact.sx";
|
||||
#import "../domain/channel.sx";
|
||||
#import "../domain/token.sx";
|
||||
#import "../domain/audit.sx";
|
||||
#import "../domain/validate.sx";
|
||||
#import "../repo/repo.sx";
|
||||
db :: #import "../repo/db.sx";
|
||||
jout :: #import "../json_out.sx";
|
||||
pl :: #import "../publish/publish.sx";
|
||||
|
||||
CleanupErr :: error {
|
||||
Load, // store database absent or unreadable
|
||||
Persist, // the pruned model could not be re-written
|
||||
}
|
||||
|
||||
CleanupOutcome :: struct {
|
||||
releases_deleted: List(string); // release ids pruned by retention
|
||||
objects_deleted: List(string); // storage keys no longer referenced
|
||||
staging_deleted: List(string); // staging file names swept
|
||||
kept_pointed_releases: List(string); // beyond retention but channel-pointed
|
||||
unlink_failed: List(string); // paths whose unlink failed post-save
|
||||
}
|
||||
|
||||
// ── directory listing (no std.fs equivalent yet) ─────────────────────
|
||||
// opendir/readdir/closedir via libc, like publish.sx's time(2) shim.
|
||||
// dirent layout is darwin-arm64: d_type at byte 20, d_name
|
||||
// (NUL-terminated, max 1024) at byte 21.
|
||||
|
||||
cl_libc :: #library "c";
|
||||
cl_opendir :: (path: [:0]u8) -> usize #foreign cl_libc "opendir";
|
||||
cl_readdir :: (dirp: usize) -> usize #foreign cl_libc "readdir";
|
||||
cl_closedir :: (dirp: usize) -> i32 #foreign cl_libc "closedir";
|
||||
|
||||
CL_DT_REG :: 8;
|
||||
|
||||
// Regular-file names in `dir` (no recursion), copied into the context
|
||||
// allocator (the dirent buffer is libc-owned and dies on the next read).
|
||||
// A missing/unopenable directory lists as empty: a store without
|
||||
// objects/ or staging/ simply has nothing to delete.
|
||||
cl_list_files :: (dir: string) -> List(string) {
|
||||
names : List(string) = .{};
|
||||
dp := cl_opendir(dir);
|
||||
if dp == 0 { return names; }
|
||||
while true {
|
||||
ent := cl_readdir(dp);
|
||||
if ent == 0 { break; }
|
||||
p : [*]u8 = xx ent;
|
||||
if p[20] == CL_DT_REG {
|
||||
n := 0;
|
||||
while p[21 + n] != 0 and n < 1023 { n += 1; }
|
||||
view := string.{ ptr = @p[21], len = n };
|
||||
names.append(db.db_dup_str(view, context.allocator), context.allocator);
|
||||
}
|
||||
}
|
||||
cl_closedir(dp);
|
||||
return names;
|
||||
}
|
||||
|
||||
cl_contains :: (l: *List(string), s: string) -> bool {
|
||||
i := 0;
|
||||
while i < l.len {
|
||||
if l.items[i] == s { return true; }
|
||||
i += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ── the cleanup transaction ──────────────────────────────────────────
|
||||
|
||||
run_cleanup :: (store_dir: string, fail_out: *jout.CliFailure) -> (CleanupOutcome, !CleanupErr) {
|
||||
if !db.store_exists(store_dir) {
|
||||
fail_out.code = "store.load";
|
||||
fail_out.message = concat("no store database (nothing published yet): ", store_dir);
|
||||
raise error.Load;
|
||||
}
|
||||
repo, le := db.load(store_dir);
|
||||
if le {
|
||||
fail_out.code = "store.load";
|
||||
fail_out.message = concat("the store database could not be loaded: ", store_dir);
|
||||
raise error.Load;
|
||||
}
|
||||
|
||||
alloc := context.allocator;
|
||||
o : CleanupOutcome = .{};
|
||||
|
||||
// Every channel's current pointer is sacrosanct, whichever channel's
|
||||
// lineage the release came from.
|
||||
guard : List(string) = .{};
|
||||
i := 0;
|
||||
while i < repo.channels.len {
|
||||
cur := repo.channels.items[i].current_release_id;
|
||||
if cur.len > 0 { guard.append(cur, alloc); }
|
||||
i += 1;
|
||||
}
|
||||
|
||||
// Per retention-bearing channel: the lineage's oldest entries beyond
|
||||
// `retention_keep` are pruned (or spared by the pointer guard). A
|
||||
// release targets exactly one channel, so lineages never overlap.
|
||||
i = 0;
|
||||
while i < repo.channels.len {
|
||||
chan := repo.channels.items[i];
|
||||
i += 1;
|
||||
if chan.retention_keep <= 0 { continue; }
|
||||
|
||||
total := 0;
|
||||
j := 0;
|
||||
while j < repo.releases.len {
|
||||
r := repo.releases.items[j];
|
||||
if r.app_id == chan.app_id and r.channel == chan.name and r.published_at > 0 { total += 1; }
|
||||
j += 1;
|
||||
}
|
||||
if total <= chan.retention_keep { continue; }
|
||||
|
||||
drop := total - chan.retention_keep; // oldest `drop` lineage entries
|
||||
seen := 0;
|
||||
j = 0;
|
||||
while j < repo.releases.len and seen < drop {
|
||||
r := repo.releases.items[j];
|
||||
j += 1;
|
||||
if !(r.app_id == chan.app_id and r.channel == chan.name and r.published_at > 0) { continue; }
|
||||
seen += 1;
|
||||
if cl_contains(@guard, r.id) {
|
||||
o.kept_pointed_releases.append(r.id, alloc);
|
||||
} else {
|
||||
o.releases_deleted.append(r.id, alloc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Storage keys the surviving artifacts still reference; any objects/
|
||||
// file outside this set is garbage.
|
||||
referenced : List(string) = .{};
|
||||
i = 0;
|
||||
while i < repo.artifacts.len {
|
||||
a := repo.artifacts.items[i];
|
||||
if !cl_contains(@o.releases_deleted, a.release_id) {
|
||||
referenced.append(a.storage_key, alloc);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
objects := cl_list_files(path_join(store_dir, "objects"));
|
||||
i = 0;
|
||||
while i < objects.len {
|
||||
if !cl_contains(@referenced, objects.items[i]) {
|
||||
o.objects_deleted.append(objects.items[i], alloc);
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
|
||||
o.staging_deleted = cl_list_files(path_join(store_dir, "staging"));
|
||||
|
||||
// Compact the model in place (order preserved): pruned releases go,
|
||||
// and their artifact rows with them.
|
||||
w := 0;
|
||||
i = 0;
|
||||
while i < repo.releases.len {
|
||||
if !cl_contains(@o.releases_deleted, repo.releases.items[i].id) {
|
||||
repo.releases.items[w] = repo.releases.items[i];
|
||||
w += 1;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
repo.releases.len = w;
|
||||
w = 0;
|
||||
i = 0;
|
||||
while i < repo.artifacts.len {
|
||||
if !cl_contains(@o.releases_deleted, repo.artifacts.items[i].release_id) {
|
||||
repo.artifacts.items[w] = repo.artifacts.items[i];
|
||||
w += 1;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
repo.artifacts.len = w;
|
||||
|
||||
// One audit event per deletion, written into the same save.
|
||||
now := pl.now_secs();
|
||||
i = 0;
|
||||
while i < o.releases_deleted.len {
|
||||
id := o.releases_deleted.items[i];
|
||||
repo.create_audit_event(AuditEvent.{
|
||||
id = concat("evt-cleanup-release-", id),
|
||||
actor = "cli", action = "release.delete", target_type = "release",
|
||||
target_id = id, metadata = "retention", created_at = now,
|
||||
});
|
||||
i += 1;
|
||||
}
|
||||
i = 0;
|
||||
while i < o.objects_deleted.len {
|
||||
key := o.objects_deleted.items[i];
|
||||
repo.create_audit_event(AuditEvent.{
|
||||
id = concat("evt-cleanup-object-", key),
|
||||
actor = "cli", action = "object.delete", target_type = "object",
|
||||
target_id = key, metadata = "unreferenced", created_at = now,
|
||||
});
|
||||
i += 1;
|
||||
}
|
||||
i = 0;
|
||||
while i < o.staging_deleted.len {
|
||||
name := o.staging_deleted.items[i];
|
||||
repo.create_audit_event(AuditEvent.{
|
||||
id = concat("evt-cleanup-staging-", name),
|
||||
actor = "cli", action = "staging.delete", target_type = "staging",
|
||||
target_id = name, metadata = "stale", created_at = now,
|
||||
});
|
||||
i += 1;
|
||||
}
|
||||
|
||||
werr := false;
|
||||
db.save(@repo, store_dir) catch { werr = true; };
|
||||
if werr {
|
||||
fail_out.code = "persist.save";
|
||||
fail_out.message = concat("the store database could not be written: ", store_dir);
|
||||
raise error.Persist;
|
||||
}
|
||||
|
||||
// Files go only after the model committed.
|
||||
i = 0;
|
||||
while i < o.objects_deleted.len {
|
||||
p := path_join(path_join(store_dir, "objects"), o.objects_deleted.items[i]);
|
||||
if !fs.delete_file(p) { o.unlink_failed.append(p, alloc); }
|
||||
i += 1;
|
||||
}
|
||||
i = 0;
|
||||
while i < o.staging_deleted.len {
|
||||
p := path_join(path_join(store_dir, "staging"), o.staging_deleted.items[i]);
|
||||
if !fs.delete_file(p) { o.unlink_failed.append(p, alloc); }
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
// ── rendering ─────────────────────────────────────────────────────────
|
||||
|
||||
cl_str_array :: (l: *List(string), alloc: Allocator) -> Value {
|
||||
arr : Array = .{};
|
||||
i := 0;
|
||||
while i < l.len {
|
||||
arr.add(.str(l.items[i]), alloc);
|
||||
i += 1;
|
||||
}
|
||||
return .array(arr);
|
||||
}
|
||||
|
||||
// `{"status":"cleaned","releases_deleted":[...],"objects_deleted":[...],
|
||||
// "staging_deleted":[...],"kept_pointed_releases":[...],
|
||||
// "unlink_failed":[...]}`.
|
||||
write_cleanup_json :: (o: *CleanupOutcome, dst: []u8) -> (i64, !JsonError) {
|
||||
gpa := GPA.init();
|
||||
root : Object = .{};
|
||||
root.put("status", .str("cleaned"), xx gpa);
|
||||
root.put("releases_deleted", cl_str_array(@o.releases_deleted, xx gpa), xx gpa);
|
||||
root.put("objects_deleted", cl_str_array(@o.objects_deleted, xx gpa), xx gpa);
|
||||
root.put("staging_deleted", cl_str_array(@o.staging_deleted, xx gpa), xx gpa);
|
||||
root.put("kept_pointed_releases", cl_str_array(@o.kept_pointed_releases, xx gpa), xx gpa);
|
||||
root.put("unlink_failed", cl_str_array(@o.unlink_failed, xx gpa), xx gpa);
|
||||
rootv : Value = .object(root);
|
||||
n := try write_to_buffer(rootv, dst);
|
||||
return n;
|
||||
}
|
||||
|
||||
cleanup_human :: (o: *CleanupOutcome) -> string {
|
||||
s := concat("cleaned: ", int_to_string(o.releases_deleted.len));
|
||||
s = concat(s, concat(" releases, ", int_to_string(o.objects_deleted.len)));
|
||||
s = concat(s, concat(" objects, ", int_to_string(o.staging_deleted.len)));
|
||||
s = concat(s, " staged files deleted");
|
||||
if o.kept_pointed_releases.len > 0 {
|
||||
s = concat(s, concat("; kept (channel-pointed): ", int_to_string(o.kept_pointed_releases.len)));
|
||||
}
|
||||
if o.unlink_failed.len > 0 {
|
||||
s = concat(s, concat("; UNLINK FAILED: ", int_to_string(o.unlink_failed.len)));
|
||||
}
|
||||
return concat(s, "\n");
|
||||
}
|
||||
267
tests/retention_cleanup.sx
Normal file
267
tests/retention_cleanup.sx
Normal file
@@ -0,0 +1,267 @@
|
||||
// Pinned acceptance for P5.3 (subplan 02 Slice 4) — retention policy +
|
||||
// `dist channel set` + `dist store cleanup`.
|
||||
//
|
||||
// Drives the BUILT `build/dist` binary (via `process.run`, like
|
||||
// release_ops.sx) through the slice's acceptance scenario:
|
||||
//
|
||||
// 1. Publish A(1.0.0), B(1.1.0), B2(1.1.1), C(1.2.0) into beta — A and
|
||||
// B carry unique payload bytes; B2 and C share one payload (one
|
||||
// content-addressed object between them) → beta points at C, the
|
||||
// store holds 3 objects.
|
||||
// 2. Cross-promote A onto stable → stable points at A (a release deep
|
||||
// in beta's lineage).
|
||||
// 3. `channel set --retention-keep 1` on beta → exit 0; dist.db row
|
||||
// records it; a `channel.update` audit event is written. Stable gets
|
||||
// `--retention-keep 5` — a longer retention than beta, per the
|
||||
// slice's "stable retained longer than beta" acceptance. A bad value
|
||||
// exits 64; an unknown channel exits 1 with `channel.unknown`.
|
||||
// 4. Seed two stale staging files, then `store cleanup`: beta's lineage
|
||||
// [A B B2 C] keeps only C (newest 1); A is beyond retention but
|
||||
// POINTED AT by stable, so it survives (`kept_pointed_releases`).
|
||||
// B and B2 are deleted with their artifact rows; B's unique object
|
||||
// is GC'd while B2's object SURVIVES (C still references those
|
||||
// bytes); staging is swept. One audit event per deletion
|
||||
// (release.delete ×2, object.delete ×1, staging.delete ×2).
|
||||
// 5. Publish D(1.3.0) into beta → beta's retention_keep is STILL 1
|
||||
// (regression: repo.publish must not reset channel fields when
|
||||
// moving the pointer). A second cleanup then prunes C (no longer
|
||||
// pointed, beyond retention) but keeps the shared object (D
|
||||
// references it). A third cleanup deletes nothing (idempotent).
|
||||
// 6. Cleanup on a store that was never published → exit 1 +
|
||||
// `store.load` JSON error.
|
||||
//
|
||||
// Store-side state is asserted by QUERYING `<store>/dist.db` through the
|
||||
// SQLite bindings and checking `<store>/objects` / `<store>/staging` on
|
||||
// the filesystem.
|
||||
#import "modules/std.sx";
|
||||
#import "modules/std/json.sx";
|
||||
process :: #import "modules/std/process.sx";
|
||||
fs :: #import "modules/std/fs.sx";
|
||||
sq :: #import "vendors/sqlite/sqlite.sx";
|
||||
|
||||
STORE :: ".sx-tmp/retention_cleanup";
|
||||
MDIR :: ".sx-tmp/retention_cleanup_m";
|
||||
|
||||
REL_A :: "rel-acme-app-1.0.0";
|
||||
REL_B :: "rel-acme-app-1.1.0";
|
||||
REL_B2 :: "rel-acme-app-1.1.1";
|
||||
REL_C :: "rel-acme-app-1.2.0";
|
||||
REL_D :: "rel-acme-app-1.3.0";
|
||||
|
||||
get :: (o: Object, key: string) -> Value {
|
||||
i := 0;
|
||||
while i < o.len {
|
||||
if o.items[i].key == key { return o.items[i].val; }
|
||||
i += 1;
|
||||
}
|
||||
process.assert(false, concat("missing json key: ", key));
|
||||
dummy : Value = .null_;
|
||||
return dummy;
|
||||
}
|
||||
get_str :: (o: Object, key: string) -> string { return get(o, key).str; }
|
||||
get_obj :: (o: Object, key: string) -> Object { return get(o, key).object; }
|
||||
get_arr :: (o: Object, key: string) -> Array { return get(o, key).array; }
|
||||
|
||||
arr_contains :: (a: Array, s: string) -> bool {
|
||||
i := 0;
|
||||
while i < a.len {
|
||||
if a.items[i].str == s { return true; }
|
||||
i += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
write_file :: (path: string, body: string) {
|
||||
cmd := concat(concat(concat("printf '%s' '", body), "' > "), path);
|
||||
process.run(cmd);
|
||||
}
|
||||
|
||||
db_open_ro :: () -> sq.Sqlite {
|
||||
c, oe := sq.Sqlite.open_v2(path_join(STORE, "dist.db"), sq.SQLITE_OPEN_READONLY);
|
||||
process.assert(!oe, "dist.db must open as a SQLite database");
|
||||
c.busy_timeout(2000);
|
||||
return c;
|
||||
}
|
||||
|
||||
// One-row TEXT scalar with up to two text bindings ("" = unbound).
|
||||
q_text :: (sql: string, p1: string, p2: string) -> string {
|
||||
c := db_open_ro();
|
||||
st, pe := c.prepare(sql);
|
||||
process.assert(!pe, concat("prepare must succeed: ", sql));
|
||||
if p1.len > 0 { st.bind_text(1, p1) catch { process.assert(false, "bind 1 failed"); }; }
|
||||
if p2.len > 0 { st.bind_text(2, p2) catch { process.assert(false, "bind 2 failed"); }; }
|
||||
rc, se := st.step();
|
||||
process.assert(!se, concat("step must succeed: ", sql));
|
||||
process.assert(rc == sq.SQLITE_ROW, concat("query must return a row: ", sql));
|
||||
out := st.column_text(0);
|
||||
st.finalize();
|
||||
c.close();
|
||||
return out;
|
||||
}
|
||||
|
||||
// One-row INTEGER scalar with up to two text bindings ("" = unbound).
|
||||
q_int :: (sql: string, p1: string, p2: string) -> i64 {
|
||||
c := db_open_ro();
|
||||
st, pe := c.prepare(sql);
|
||||
process.assert(!pe, concat("prepare must succeed: ", sql));
|
||||
if p1.len > 0 { st.bind_text(1, p1) catch { process.assert(false, "bind 1 failed"); }; }
|
||||
if p2.len > 0 { st.bind_text(2, p2) catch { process.assert(false, "bind 2 failed"); }; }
|
||||
rc, se := st.step();
|
||||
process.assert(!se, concat("step must succeed: ", sql));
|
||||
process.assert(rc == sq.SQLITE_ROW, concat("query must return a row: ", sql));
|
||||
out := st.column_int64(0);
|
||||
st.finalize();
|
||||
c.close();
|
||||
return out;
|
||||
}
|
||||
|
||||
// Write a manifest + payload pair into MDIR and publish it (artifact
|
||||
// paths resolve relative to the manifest file).
|
||||
publish :: (version: string, payload_name: string, payload_bytes: string) {
|
||||
ppath := path_join(MDIR, payload_name);
|
||||
if !fs.exists(ppath) { write_file(ppath, payload_bytes); }
|
||||
m := concat("{\"app\":\"acme-app\",\"version\":\"", version);
|
||||
m = concat(m, "\",\"channel\":\"beta\",\"artifacts\":[{\"platform\":\"android_apk\",\"path\":\"");
|
||||
m = concat(m, concat(payload_name, "\"}]}"));
|
||||
mpath := path_join(MDIR, concat(concat("m-", version), ".json"));
|
||||
write_file(mpath, m);
|
||||
cmd := concat(concat("build/dist ci publish --manifest ", mpath), concat(concat(" --local-store ", STORE), " --json 2>/dev/null"));
|
||||
r := process.run(cmd);
|
||||
process.assert(r != null and r!.exit_code == 0, concat("publish must exit 0: ", version));
|
||||
}
|
||||
|
||||
cleanup_cmd :: () -> string {
|
||||
return concat(concat("build/dist store cleanup --local-store ", STORE), " --json 2>/dev/null");
|
||||
}
|
||||
|
||||
beta_retention :: () -> i64 {
|
||||
return q_int("SELECT retention_keep FROM channels WHERE name = ?1", "beta", "");
|
||||
}
|
||||
|
||||
main :: () -> i32 {
|
||||
gpa := GPA.init();
|
||||
arena := Arena.init(xx gpa, 1 << 20);
|
||||
defer arena.deinit();
|
||||
|
||||
process.run(concat("rm -rf ", STORE));
|
||||
process.run(concat("rm -rf ", MDIR));
|
||||
process.run(concat("mkdir -p ", MDIR));
|
||||
|
||||
// ── 1+2. Lineage [A B B2 C] on beta (B2 and C share bytes), A
|
||||
// cross-promoted onto stable ────────────────────────────────
|
||||
publish("1.0.0", "payload-a.apk", "unique-bytes-of-A");
|
||||
publish("1.1.0", "payload-b.apk", "unique-bytes-of-B");
|
||||
publish("1.1.1", "payload-shared.apk", "shared-bytes-B2-C-D");
|
||||
publish("1.2.0", "payload-shared.apk", "shared-bytes-B2-C-D");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM releases", "", "") == 4, "four releases published");
|
||||
rp := process.run(concat(concat("build/dist release promote --app acme-app --channel stable --release ", REL_A),
|
||||
concat(concat(" --local-store ", STORE), " --json 2>/dev/null")));
|
||||
process.assert(rp != null and rp!.exit_code == 0, "promote A onto stable must exit 0");
|
||||
key_b := q_text("SELECT sha256 FROM artifacts WHERE release_id = ?1", REL_B, "");
|
||||
key_shared := q_text("SELECT sha256 FROM artifacts WHERE release_id = ?1", REL_C, "");
|
||||
print(" lineage [A B B2 C] on beta; stable -> A; 3 objects stored\n");
|
||||
|
||||
// ── 3. channel set: beta keeps 1, stable keeps 5; failure paths ──
|
||||
cs := process.run(concat(concat("build/dist channel set --app acme-app --channel beta --retention-keep 1 --local-store ", STORE), " --json 2>/dev/null"));
|
||||
process.assert(cs != null and cs!.exit_code == 0, "channel set beta must exit 0");
|
||||
cv, ce := parse(cs!.stdout, xx arena);
|
||||
if ce { process.assert(false, "channel set stdout must be one JSON object"); return 1; }
|
||||
co := cv.object;
|
||||
process.assert(get_str(co, "status") == "updated", "channel set json status");
|
||||
process.assert(get(get_obj(co, "channel"), "retention_keep").int_ == 1, "channel set json retention_keep");
|
||||
process.assert(beta_retention() == 1, "dist.db: beta retention_keep = 1");
|
||||
|
||||
cs5 := process.run(concat(concat("build/dist channel set --app acme-app --channel stable --retention-keep 5 --local-store ", STORE), " --json 2>/dev/null"));
|
||||
process.assert(cs5 != null and cs5!.exit_code == 0, "channel set stable must exit 0");
|
||||
process.assert(q_int("SELECT retention_keep FROM channels WHERE name = ?1", "stable", "") == 5,
|
||||
"dist.db: stable retains longer than beta (5 > 1)");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM audit_events WHERE actor = ?1 AND action = ?2", "cli", "channel.update") == 2,
|
||||
"each channel set recorded a cli channel.update audit event");
|
||||
|
||||
bad := process.run(concat(concat("build/dist channel set --app acme-app --channel beta --retention-keep abc --local-store ", STORE), " --json 2>/dev/null"));
|
||||
process.assert(bad != null and bad!.exit_code == 64, "non-numeric --retention-keep is a usage error (64)");
|
||||
unk := process.run(concat(concat("build/dist channel set --app acme-app --channel nope --retention-keep 1 --local-store ", STORE), " --json 2>/dev/null"));
|
||||
process.assert(unk != null and unk!.exit_code == 1, "unknown channel must exit 1");
|
||||
uv, ue := parse(unk!.stdout, xx arena);
|
||||
if ue { process.assert(false, "unknown-channel stdout must be one JSON object"); return 1; }
|
||||
process.assert(get_str(get_obj(uv.object, "error"), "code") == "channel.unknown", "unknown-channel json names the code");
|
||||
print(" channel set: beta keep 1, stable keep 5, audited; bad value 64, unknown channel 1\n");
|
||||
|
||||
// ── 4. cleanup: B+B2 pruned, A spared (pointed), B's object GC'd,
|
||||
// shared object survives, staging swept, all audited ─────────
|
||||
process.run(concat("mkdir -p ", path_join(STORE, "staging")));
|
||||
write_file(path_join(STORE, "staging/incoming-7"), "half-written upload");
|
||||
write_file(path_join(STORE, "staging/deadbeef"), "stale staged bytes");
|
||||
|
||||
r1 := process.run(cleanup_cmd());
|
||||
process.assert(r1 != null and r1!.exit_code == 0, "cleanup must exit 0");
|
||||
v1, e1 := parse(r1!.stdout, xx arena);
|
||||
if e1 { process.assert(false, "cleanup stdout must be one JSON object"); return 1; }
|
||||
o1 := v1.object;
|
||||
process.assert(get_str(o1, "status") == "cleaned", "cleanup json status");
|
||||
rd := get_arr(o1, "releases_deleted");
|
||||
process.assert(rd.len == 2 and arr_contains(rd, REL_B) and arr_contains(rd, REL_B2),
|
||||
"cleanup deleted exactly B and B2");
|
||||
kp := get_arr(o1, "kept_pointed_releases");
|
||||
process.assert(kp.len == 1 and arr_contains(kp, REL_A),
|
||||
"A is beyond retention but spared: stable points at it");
|
||||
od := get_arr(o1, "objects_deleted");
|
||||
process.assert(od.len == 1 and arr_contains(od, key_b), "only B's unreferenced object was GC'd");
|
||||
sd := get_arr(o1, "staging_deleted");
|
||||
process.assert(sd.len == 2 and arr_contains(sd, "incoming-7") and arr_contains(sd, "deadbeef"),
|
||||
"both stale staging files swept");
|
||||
process.assert(get_arr(o1, "unlink_failed").len == 0, "no unlink failures");
|
||||
|
||||
process.assert(q_int("SELECT COUNT(*) FROM releases", "", "") == 2, "releases left: A and C");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM releases WHERE id = ?1", REL_A, "") == 1, "A survived");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM releases WHERE id = ?1", REL_C, "") == 1, "C survived");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM artifacts", "", "") == 2, "pruned releases lost their artifact rows");
|
||||
process.assert(q_text("SELECT current_release_id FROM channels WHERE name = ?1", "stable", "") == REL_A, "stable still -> A");
|
||||
process.assert(q_text("SELECT current_release_id FROM channels WHERE name = ?1", "beta", "") == REL_C, "beta still -> C");
|
||||
process.assert(!fs.exists(path_join(STORE, concat("objects/", key_b))), "B's object bytes are gone");
|
||||
process.assert(fs.exists(path_join(STORE, concat("objects/", key_shared))), "the shared object survived (C references it)");
|
||||
process.assert(!fs.exists(path_join(STORE, "staging/incoming-7")), "staging file 1 gone");
|
||||
process.assert(!fs.exists(path_join(STORE, "staging/deadbeef")), "staging file 2 gone");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM audit_events WHERE action = ?1", "release.delete", "") == 2, "2 release.delete audit events");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM audit_events WHERE action = ?1 AND target_id = ?2", "object.delete", key_b) == 1, "object.delete audit names B's key");
|
||||
process.assert(q_int("SELECT COUNT(*) FROM audit_events WHERE action = ?1", "staging.delete", "") == 2, "2 staging.delete audit events");
|
||||
print(" cleanup: B+B2 pruned, A spared via stable, shared object kept, staging swept, audited\n");
|
||||
|
||||
// ── 5. retention survives publish (repo.publish regression), then
|
||||
// a second cleanup prunes C and an idle third does nothing ───
|
||||
publish("1.3.0", "payload-shared.apk", "shared-bytes-B2-C-D");
|
||||
process.assert(beta_retention() == 1, "publishing into beta must NOT reset retention_keep");
|
||||
|
||||
r2 := process.run(cleanup_cmd());
|
||||
process.assert(r2 != null and r2!.exit_code == 0, "second cleanup must exit 0");
|
||||
v2, e2 := parse(r2!.stdout, xx arena);
|
||||
if e2 { process.assert(false, "second cleanup stdout must be one JSON object"); return 1; }
|
||||
rd2 := get_arr(v2.object, "releases_deleted");
|
||||
process.assert(rd2.len == 1 and arr_contains(rd2, REL_C), "second cleanup prunes C (beta -> D now)");
|
||||
process.assert(get_arr(v2.object, "objects_deleted").len == 0, "shared object survives: D references it");
|
||||
process.assert(fs.exists(path_join(STORE, concat("objects/", key_shared))), "shared object bytes still present");
|
||||
process.assert(q_text("SELECT current_release_id FROM channels WHERE name = ?1", "beta", "") == REL_D, "beta -> D");
|
||||
|
||||
r3 := process.run(cleanup_cmd());
|
||||
process.assert(r3 != null and r3!.exit_code == 0, "third cleanup must exit 0");
|
||||
v3, e3 := parse(r3!.stdout, xx arena);
|
||||
if e3 { process.assert(false, "third cleanup stdout must be one JSON object"); return 1; }
|
||||
process.assert(get_arr(v3.object, "releases_deleted").len == 0
|
||||
and get_arr(v3.object, "objects_deleted").len == 0
|
||||
and get_arr(v3.object, "staging_deleted").len == 0,
|
||||
"an idle cleanup deletes nothing (idempotent)");
|
||||
print(" retention survives publish; second cleanup prunes C, third is a no-op\n");
|
||||
|
||||
// ── 6. cleanup on a never-published store fails loudly ───────────
|
||||
rn := process.run("build/dist store cleanup --local-store .sx-tmp/retention_cleanup_none --json 2>/dev/null");
|
||||
process.assert(rn != null and rn!.exit_code == 1, "cleanup on a missing store must exit 1");
|
||||
nv, ne := parse(rn!.stdout, xx arena);
|
||||
if ne { process.assert(false, "missing-store stdout must be one JSON object"); return 1; }
|
||||
process.assert(get_str(get_obj(nv.object, "error"), "code") == "store.load", "missing-store json names the code");
|
||||
print(" cleanup on a missing store: exit 1 + store.load\n");
|
||||
|
||||
process.run(concat("rm -rf ", STORE));
|
||||
process.run(concat("rm -rf ", MDIR));
|
||||
print("retention_cleanup: ALL CASES PASS\n");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user