lab/distribution
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P3.4a-001: ci publish loads existing db.json (cross-invocation persistence)

Browse Source 
`dist ci publish` now seeds the Repo from a pre-existing <store>/db.json
before find-or-create, so separate CLI invocations share state: a new
version accumulates under the single found app, and re-publishing the same
release id is rejected by the P2.3 integrity transaction (db.json left
unchanged). An absent db.json still starts empty. The loaded model grows
through its owning allocator (context.allocator), per the long-lived rule.

Wiring db.load into the dist program (which already links manifest.sx)
exposed two latent issues, both fixed:
- db.sx's load-path helpers (dup_str/obj_find/req_obj/req_arr/
  artifact_from_json) collided by name with manifest.sx's same-named
  helpers; sx resolves bare top-level names across the whole program, so
  load_into bound to manifest's versions and failed the LoadErr error-set
  check. Renamed db.sx's five helpers with a db_ prefix (load-path only;
  save path and public API untouched).
- publish's `existing!.id` (only reachable once an app is found, i.e. never
  before this change) read garbage: sx miscompiles postfix-`!` chained with
  `.field`. Bound the unwrap to a local first, matching the codebase idiom.

tests/publish_persist.sx drives build/dist twice into one store: publish A,
then a different version B accumulates (two releases, one app, both objects),
then re-publishing A's id fails and leaves db.json unchanged. Fails on the
pre-fix write-only persistence, passes after.
This commit is contained in:
agra
2026-06-06 06:41:11 +03:00
parent 622ad91e26
commit ea2cf14f48
3 changed files with 238 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/publish/publish.sx
View File

@@ -5,13 +5,15 @@
// manifest (P3.2) -> store (P2.2) -> common validation (P3.3) ->
// repository transaction + audit (P2.3) -> db.json persistence (P2.3)
//
// `run_publish(manifest_path, store_dir)` validates the manifest, finds or
// creates the app, drafts a release, content-addresses every artifact into
// `run_publish(manifest_path, store_dir)` validates the manifest, LOADS any
// prior `<store>/db.json` so separate invocations share state (a new version
// accumulates; a duplicate release id is rejected), finds or creates the app,
// drafts a release, content-addresses every artifact into
// `<store>/objects/<sha256>`, validates each stored file, commits the whole
// aggregate through the integrity-checked repo transaction (channel
// promotion included), records an audit event per upload / publish /
// promotion, persists `<store>/db.json`, and returns a `PublishOutcome` the
// CLI renders as stable JSON or a human summary.
// promotion, persists the merged `<store>/db.json`, and returns a
// `PublishOutcome` the CLI renders as stable JSON or a human summary.
//
// DECLARED-vs-DERIVED EXPECTATIONS (PO ruling): a manifest artifact may
// DECLARE `size` / `sha256`; when it does, that value is the expectation the
@@ -59,7 +61,7 @@ c_getcwd :: (buf: [*]u8, size: usize) -> *u8 #foreign cstd "getcwd";
// Store — an artifact's bytes could not be content-addressed.
// Validation — a stored artifact failed the common validation pass.
// Transaction — the repo's integrity-checked publish rejected the aggregate.
// Persist — db.json could not be written.
// Persist — db.json could not be loaded at startup or written at the end.
PublishError :: error {
Manifest,
Store,
@@ -135,7 +137,20 @@ run_publish :: (manifest_path: string, store_dir: string) -> (PublishOutcome, !P
abs := abs_store(store_dir);
now := now_secs();
// Seed the Repo from any prior state so separate CLI invocations SHARE
// state through the store: a pre-existing `<store>/db.json` is loaded so
// find-or-create sees earlier apps and the integrity transaction sees
// earlier releases. A new version then ACCUMULATES (the app is found, not
// duplicated); re-publishing the SAME release id is rejected as a
// duplicate by the transaction. An absent db.json starts empty. The loaded
// model grows through its own owning allocator (`context.allocator`, the
// process-lifetime default), per the long-lived-container rule.
repo := Repo.init();
if exists(path_join(store_dir, "db.json")) {
loaded, le := db.load(store_dir);
if le { raise error.Persist; }
repo = loaded;
}
st := Store.init(store_dir);
// 2. Find or create the app (keyed by slug).
@@ -150,7 +165,8 @@ run_publish :: (manifest_path: string, store_dir: string) -> (PublishOutcome, !P
created_at = now, updated_at = now,
});
} else {
app_id = existing!.id;
found := existing!;
app_id = found.id;
}
// 3. Draft the release for this version/channel.

55
src/repo/db.sx
View File

@@ -226,17 +226,24 @@ save :: (self: *Repo, root_dir: string) -> !LoadErr {
}
// ── read-back helpers (strict; copy strings into `alloc`) ────────────
// These carry a `db_` prefix because the `dist` program links this module
// alongside `manifest.sx`, which declares its own same-purpose `dup_str` /
// `obj_find` / `req_obj` / `req_arr` / `artifact_from_json`. sx resolves a
// bare top-level name across the WHOLE program (see the `jsonp` note above),
// so an unprefixed name here would bind to manifest's version and mismatch
// this module's `LoadErr` error set. The prefix keeps the load path bound to
// its own helpers.
// Copy `s` into `alloc`-owned, null-terminated storage so it survives the
// parse scratch / source buffer being freed.
dup_str :: (s: string, alloc: Allocator) -> string {
db_dup_str :: (s: string, alloc: Allocator) -> string {
raw : [*]u8 = xx alloc.alloc(s.len + 1);
if s.len > 0 { memcpy(raw, s.ptr, s.len); }
raw[s.len] = 0;
return string.{ ptr = raw, len = s.len };
}
obj_find :: (o: Object, key: string) -> ?Value {
db_obj_find :: (o: Object, key: string) -> ?Value {
i := 0;
while i < o.len {
if o.items[i].key == key { return o.items[i].val; }
@@ -247,17 +254,17 @@ obj_find :: (o: Object, key: string) -> ?Value {
// Required string field, copied into `alloc`.
req_str :: (o: Object, key: string, alloc: Allocator) -> (string, !LoadErr) {
v := obj_find(o, key);
v := db_obj_find(o, key);
if v == null { raise error.BadShape; }
val := v!;
if val != .str { raise error.BadShape; }
return dup_str(val.str, alloc);
return db_dup_str(val.str, alloc);
}
// Required string field as a borrowed VIEW (for enum-name parsing only;
// not stored, so no copy needed).
req_str_view :: (o: Object, key: string) -> (string, !LoadErr) {
v := obj_find(o, key);
v := db_obj_find(o, key);
if v == null { raise error.BadShape; }
val := v!;
if val != .str { raise error.BadShape; }
@@ -265,22 +272,22 @@ req_str_view :: (o: Object, key: string) -> (string, !LoadErr) {
}
req_int :: (o: Object, key: string) -> (s64, !LoadErr) {
v := obj_find(o, key);
v := db_obj_find(o, key);
if v == null { raise error.BadShape; }
val := v!;
if val != .int_ { raise error.BadShape; }
return val.int_;
}
req_arr :: (o: Object, key: string) -> (Array, !LoadErr) {
v := obj_find(o, key);
db_req_arr :: (o: Object, key: string) -> (Array, !LoadErr) {
v := db_obj_find(o, key);
if v == null { raise error.BadShape; }
val := v!;
if val != .array { raise error.BadShape; }
return val.array;
}
req_obj :: (v: Value) -> (Object, !LoadErr) {
db_req_obj :: (v: Value) -> (Object, !LoadErr) {
if v != .object { raise error.BadShape; }
return v.object;
}
@@ -291,10 +298,10 @@ app_from_json :: (o: Object, alloc: Allocator) -> (App, !LoadErr) {
a.id = try req_str(o, "id", alloc);
a.slug = try req_str(o, "slug", alloc);
a.display_name = try req_str(o, "display_name", alloc);
bids := try req_arr(o, "bundle_ids");
bids := try db_req_arr(o, "bundle_ids");
i := 0;
while i < bids.len {
bo := try req_obj(bids.items[i]);
bo := try db_req_obj(bids.items[i]);
p := try platform_from(try req_str_view(bo, "platform"));
bid : BundleId = .{ platform = p, value = try req_str(bo, "value", alloc) };
a.bundle_ids.append(bid, alloc);
@@ -321,7 +328,7 @@ release_from_json :: (o: Object, alloc: Allocator) -> (Release, !LoadErr) {
return r;
}
artifact_from_json :: (o: Object, alloc: Allocator) -> (Artifact, !LoadErr) {
db_artifact_from_json :: (o: Object, alloc: Allocator) -> (Artifact, !LoadErr) {
a : Artifact = .{};
a.id = try req_str(o, "id", alloc);
a.app_id = try req_str(o, "app_id", alloc);
@@ -366,44 +373,44 @@ load_into :: (repo: *Repo, bytes: string, scratch: Allocator) -> !LoadErr {
root_val, pe := jsonp.parse(bytes, scratch);
if pe { raise error.Parse; }
ro := try req_obj(root_val);
ro := try db_req_obj(root_val);
apps_arr := try req_arr(ro, "apps");
apps_arr := try db_req_arr(ro, "apps");
i := 0;
while i < apps_arr.len {
ao := try req_obj(apps_arr.items[i]);
ao := try db_req_obj(apps_arr.items[i]);
repo.create_app(try app_from_json(ao, oa));
i += 1;
}
rel_arr := try req_arr(ro, "releases");
rel_arr := try db_req_arr(ro, "releases");
i = 0;
while i < rel_arr.len {
o := try req_obj(rel_arr.items[i]);
o := try db_req_obj(rel_arr.items[i]);
repo.create_release(try release_from_json(o, oa));
i += 1;
}
art_arr := try req_arr(ro, "artifacts");
art_arr := try db_req_arr(ro, "artifacts");
i = 0;
while i < art_arr.len {
o := try req_obj(art_arr.items[i]);
repo.create_artifact(try artifact_from_json(o, oa));
o := try db_req_obj(art_arr.items[i]);
repo.create_artifact(try db_artifact_from_json(o, oa));
i += 1;
}
chan_arr := try req_arr(ro, "channels");
chan_arr := try db_req_arr(ro, "channels");
i = 0;
while i < chan_arr.len {
o := try req_obj(chan_arr.items[i]);
o := try db_req_obj(chan_arr.items[i]);
repo.create_channel(try channel_from_json(o, oa));
i += 1;
}
ev_arr := try req_arr(ro, "audit_events");
ev_arr := try db_req_arr(ro, "audit_events");
i = 0;
while i < ev_arr.len {
o := try req_obj(ev_arr.items[i]);
o := try db_req_obj(ev_arr.items[i]);
repo.create_audit_event(try audit_from_json(o, oa));
i += 1;
}

185
tests/publish_persist.sx Normal file
View File

@@ -0,0 +1,185 @@
// Regression for P3.4a-001 — `dist ci publish` must LOAD an existing
// `<store>/db.json` before publishing, so separate CLI invocations SHARE
// state through the store (not start from an empty Repo and clobber it).
//
// Drives the BUILT `build/dist` binary (via `process.run`, like
// publish_happy.sx) twice into ONE store and asserts cross-invocation
// persistence:
//
// 1. Publish version A (1.2.3) into a fresh store → db.json has the release.
// 2. Publish a DIFFERENT version B (1.2.4) of the SAME app into the SAME
// store → exit 0, and db.json now records BOTH releases under ONE app
// (the app is FOUND, not duplicated); the channel points at the latest
// release B; both content-addressed objects exist.
// 3. Re-publishing the SAME release id (A again) into the same store FAILS
// (exit != 0 — the P2.3 integrity transaction rejects the duplicate
// release id) and leaves db.json UNCHANGED (still two releases).
//
// FAIL-BEFORE / PASS-AFTER: against the pre-fix publish (which never reads
// db.json and so begins every invocation from an EMPTY Repo) step 2 CLOBBERS
// A — db.json ends with one release, not two — and step 3 "succeeds" (exit 0)
// and overwrites. Both assertions fail. After the load-then-merge fix they
// pass. Fresh store per run.
#import "modules/std.sx";
#import "modules/std/json.sx";
process :: #import "modules/process.sx";
fs :: #import "modules/fs.sx";
STORE :: ".sx-tmp/publish_persist";
MDIR :: ".sx-tmp/publish_persist_m";
A_PATH :: ".sx-tmp/publish_persist_m/a.json";
B_PATH :: ".sx-tmp/publish_persist_m/b.json";
// Two manifests for the SAME app/channel/fixtures, differing only in version.
// Artifact paths resolve relative to the manifest's own directory (MDIR), so
// `../../examples/fixtures/...` reaches the repo's committed fixtures.
MANIFEST_A :: "{\"app\":\"acme-app\",\"version\":\"1.2.3\",\"channel\":\"stable\",\"artifacts\":[{\"platform\":\"android_apk\",\"path\":\"../../examples/fixtures/acme-1.2.3-android.apk\"},{\"platform\":\"ios\",\"path\":\"../../examples/fixtures/acme-1.2.3-ios.ipa\"}]}";
MANIFEST_B :: "{\"app\":\"acme-app\",\"version\":\"1.2.4\",\"channel\":\"stable\",\"artifacts\":[{\"platform\":\"android_apk\",\"path\":\"../../examples/fixtures/acme-1.2.3-android.apk\"},{\"platform\":\"ios\",\"path\":\"../../examples/fixtures/acme-1.2.3-ios.ipa\"}]}";
// Fetch a member value by key, asserting presence (the publish/db output is a
// fixed shape, so an absent key is a hard failure).
get :: (o: Object, key: string) -> Value {
i := 0;
while i < o.len {
if o.items[i].key == key { return o.items[i].val; }
i += 1;
}
process.assert(false, concat("missing json key: ", key));
dummy : Value = .null_;
return dummy;
}
get_str :: (o: Object, key: string) -> string { return get(o, key).str; }
get_obj :: (o: Object, key: string) -> Object { return get(o, key).object; }
get_arr :: (o: Object, key: string) -> Array { return get(o, key).array; }
// Count audit events whose "action" equals `action`.
count_action :: (events: Array, action: string) -> s64 {
c : s64 = 0;
i := 0;
while i < events.len {
eo := events.items[i].object;
if get_str(eo, "action") == action { c += 1; }
i += 1;
}
return c;
}
// True iff `releases` (a db.json array) contains a release with id `id`.
has_release :: (releases: Array, id: string) -> bool {
i := 0;
while i < releases.len {
if get_str(releases.items[i].object, "id") == id { return true; }
i += 1;
}
return false;
}
// `build/dist ci publish` for `mpath` into the shared store, JSON mode,
// stderr suppressed so stdout stays pure for the parser.
publish_cmd :: (mpath: string) -> string {
c := concat("build/dist ci publish --manifest ", mpath);
c = concat(c, concat(" --local-store ", STORE));
return concat(c, " --json 2>/dev/null");
}
// Write `body` to `path` via the shell (single-quoted, so the JSON's double
// quotes pass through literally).
write_file :: (path: string, body: string) {
cmd := concat(concat(concat("printf '%s' '", body), "' > "), path);
process.run(cmd);
}
// Parse `<STORE>/db.json` into its root object (re-read fresh each call).
load_db :: (scratch: Allocator) -> Object {
db_bytes := fs.read_file(path_join(STORE, "db.json"));
process.assert(db_bytes != null, "db.json must exist under the store");
dv, de := parse(db_bytes!, scratch);
if de { process.assert(false, "db.json must be valid JSON"); dummy : Object = .{}; return dummy; }
return dv.object;
}
main :: () -> s32 {
gpa := GPA.init();
arena := Arena.init(xx gpa, 1 << 20);
defer arena.deinit();
// Fresh store + manifest dir, even after a crashed prior run.
process.run(concat("rm -rf ", STORE));
process.run(concat("rm -rf ", MDIR));
process.run(concat("mkdir -p ", MDIR));
write_file(A_PATH, MANIFEST_A);
write_file(B_PATH, MANIFEST_B);
rel_a := "rel-acme-app-1.2.3";
rel_b := "rel-acme-app-1.2.4";
// ── 1. Publish version A into the fresh store ───────────────────────
ra := process.run(publish_cmd(A_PATH));
process.assert(ra != null, "spawn publish A failed");
res_a := ra!;
process.assert(res_a.exit_code == 0, "publish A must exit 0");
va, ea := parse(res_a.stdout, xx arena);
if ea { process.assert(false, "publish A stdout must be one JSON object"); return 1; }
process.assert(get_str(get_obj(va.object, "release"), "id") == rel_a, "A release id");
db1 := load_db(xx arena);
process.assert(get_arr(db1, "releases").len == 1, "after A: db has one release");
process.assert(get_arr(db1, "apps").len == 1, "after A: db has one app");
print(" A published; db has 1 release\n");
// ── 2. Publish a DIFFERENT version B into the SAME store ────────────
rb := process.run(publish_cmd(B_PATH));
process.assert(rb != null, "spawn publish B failed");
res_b := rb!;
process.assert(res_b.exit_code == 0, "publish B must exit 0 (accumulates)");
vb, eb := parse(res_b.stdout, xx arena);
if eb { process.assert(false, "publish B stdout must be one JSON object"); return 1; }
process.assert(get_str(get_obj(vb.object, "release"), "id") == rel_b, "B release id");
db2 := load_db(xx arena);
// The crux: BOTH releases under ONE app (app found, not duplicated).
db2_rels := get_arr(db2, "releases");
process.assert(db2_rels.len == 2, "after B: db records BOTH releases (no clobber)");
process.assert(has_release(db2_rels, rel_a), "after B: release A still present");
process.assert(has_release(db2_rels, rel_b), "after B: release B present");
process.assert(get_arr(db2, "apps").len == 1, "after B: still ONE app (found, not duplicated)");
// Four artifacts (two per release); channel promoted to the latest (B).
process.assert(get_arr(db2, "artifacts").len == 4, "after B: four artifacts (two per release)");
db2_chans := get_arr(db2, "channels");
process.assert(db2_chans.len == 1, "after B: one channel");
process.assert(get_str(db2_chans.items[0].object, "current_release_id") == rel_b,
"after B: channel promoted to the latest release");
// Each artifact's content-addressed object exists on disk.
db2_arts := get_arr(db2, "artifacts");
k := 0;
while k < db2_arts.len {
sha := get_str(db2_arts.items[k].object, "sha256");
process.assert(fs.exists(path_join(STORE, concat("objects/", sha))),
"after B: object exists at objects/<sha256>");
k += 1;
}
// Audit accumulated across both publishes (>= 2 publish events).
process.assert(count_action(get_arr(db2, "audit_events"), "release.publish") == 2,
"after B: one publish event per release");
print(" B accumulated; db has 2 releases under 1 app\n");
// ── 3. Re-publish the SAME release id (A) → duplicate is rejected ───
rdup := process.run(publish_cmd(A_PATH));
process.assert(rdup != null, "spawn duplicate publish failed");
res_dup := rdup!;
process.assert(res_dup.exit_code != 0, "re-publishing the same release id must FAIL (duplicate)");
db3 := load_db(xx arena);
process.assert(get_arr(db3, "releases").len == 2, "after duplicate: db UNCHANGED (still two releases)");
process.assert(get_arr(db3, "apps").len == 1, "after duplicate: still one app");
print(" duplicate release id rejected; db unchanged\n");
process.run(concat("rm -rf ", STORE));
process.run(concat("rm -rf ", MDIR));
print("publish_persist: ALL CASES PASS\n");
return 0;
}