lab/distribution
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P3.4a: dist ci publish happy path + persistence (manifest->store->validate->publish->db.json->JSON)

Browse Source 
Real local publish success pipeline replacing the ci-publish stub: validate manifest,
find/create app + draft release, per-artifact content-address store (P2.2) + common
validation (P3.3) with optional manifest-declared size/sha256 (PO ruling), publish via
the repo integrity transaction with channel promotion + audit events (P2.3), persist
db.json (P2.3), emit stable JSON (release id, artifact ids, sha256, file:// URLs) with
--json purity. make publish-example target + tests/publish_happy.sx (fail-before/pass-after).

Salvaged from a worker that completed the work (make test 10/10) but hit the 50-min wall
before committing; manager-verified at ground truth (make test green, make publish-example
exit 0, stored object re-hashes to its key via shasum, db.json records release/2 artifacts/
channel/4 audit events).
This commit is contained in:
agra
2026-06-06 05:59:38 +03:00
parent 20d520d08c
commit 622ad91e26
7 changed files with 600 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
src/dist.sx
View File

@@ -2,29 +2,30 @@
// dist.sx — the `dist` distribution CLI entry point (subplan 03, Slice 1).
//
// Wires the real process argv (via `std.cli`'s `os_args`) to subcommand
// handlers through `cli.parse`. The three groups/commands are dispatched
// to STUB handlers for now — they acknowledge and emit a known result; the
// real publish/promote/rollback logic lands in P3.4/P3.5.
// handlers through `cli.parse`.
//
// dist ci publish
// dist release promote
// dist release rollback
// dist ci publish REAL local publish pipeline (P3.4a) — see publish.sx
// dist release promote STUB (real logic lands in P3.5)
// dist release rollback STUB (real logic lands in P3.5)
//
// EXIT-CODE CONTRACT (sysexits, via std.cli): success ends with
// `exit_ok()` (EX_OK = 0); a no-command / unknown-or-missing
// group/command/flag ends with `exit_usage()` (EX_USAGE = 64). An
// explicit `-h`/`--help` is not an error and ends 0.
// explicit `-h`/`--help` is not an error and ends 0. A failed publish also
// ends `exit_usage()` for now (the failure/abort contract lands in P3.4b).
//
// `--json` PURITY: every command accepts the reserved global `--json`
// flag (surfaced by the parser as `parsed.json`). In json mode stdout
// carries ONLY the machine-readable JSON object (emitted via `std.json`,
// isolated in `json_out.sx`); ALL human-readable text (help, progress,
// errors) goes to stderr. In non-json mode human text on stdout is fine.
// isolated in `json_out.sx` / `publish.sx`); ALL human-readable text (help,
// progress, errors) goes to stderr. In non-json mode human text on stdout
// is fine.
// =====================================================================
#import "modules/std.sx";
#import "modules/std/cli.sx";
jout :: #import "json_out.sx";
pl :: #import "publish/publish.sx";
// Direct stderr writer (fd 2), so human help/usage/progress never lands on
// stdout's data stream. `out` (std builtin) targets stdout (fd 1).
@@ -39,7 +40,7 @@ emit_human :: (s: string, json_mode: bool) {
if json_mode { eputs(s); } else { out(s); }
}
HELP :: "dist — application distribution CLI\n\nUsage:\n dist <group> <command> [flags]\n\nGroups & commands:\n ci\n ci publish publish a release from CI (stub)\n release\n release promote promote a release onto a channel (stub)\n release rollback roll a channel back to a prior release (stub)\n\nGlobal flags:\n --json emit machine-readable JSON on stdout; human text to stderr\n -h, --help show this help and exit\n\nExit codes:\n 0 success\n 64 usage error (no command, or an unknown/missing command or flag)\n";
HELP :: "dist — application distribution CLI\n\nUsage:\n dist <group> <command> [flags]\n\nGroups & commands:\n ci\n ci publish publish a release from CI\n --manifest <path> publish manifest (dist.json) to read\n --local-store <dir> local artifact store + db.json directory\n release\n release promote promote a release onto a channel (stub)\n release rollback roll a channel back to a prior release (stub)\n\nGlobal flags:\n --json emit machine-readable JSON on stdout; human text to stderr\n -h, --help show this help and exit\n\nExit codes:\n 0 success\n 64 usage error (no command, or an unknown/missing command or flag)\n";
// True if `name` appears as a token in `args`.
has_flag :: (args: []string, name: string) -> bool {
@@ -61,13 +62,45 @@ error_phrase :: (e: CliError) -> string {
return "usage error";
}
// ── Stub handlers ────────────────────────────────────────────────────
// Honest stubs: acknowledge the command and emit a known result. NO real
// publish/promote/rollback logic (that is P3.4/P3.5).
// ── Handlers ─────────────────────────────────────────────────────────
// `dist ci publish` — the real local publish pipeline (P3.4a). Runs the
// publish, then renders the outcome: in json mode a single stable JSON
// object on stdout with the human progress note on stderr (the `--json`
// purity contract); otherwise a readable summary on stdout. A failed
// publish prints a diagnostic to stderr and ends with EX_USAGE.
handle_ci_publish :: (p: *Parsed, json_mode: bool) {
ack("ci publish", json_mode);
manifest_path := p.value_of("manifest");
store_dir := p.value_of("local-store");
o, e := pl.run_publish(manifest_path, store_dir);
if e {
tag : u32 = xx e;
eputs(concat(concat("dist: ci publish failed: ", error_tag_name(tag)), "\n"));
exit_usage();
}
if !e {
if !json_mode {
out(pl.human_summary(@o));
return;
}
eputs("dist: ci publish ok\n");
raw : [16384]u8 = ---;
werr := false;
n := pl.write_json(@o, string.{ ptr = @raw[0], len = 16384 }) catch { werr = true; 0 };
if werr {
eputs("dist: internal error: JSON serialization failed\n");
exit_usage();
}
out(string.{ ptr = @raw[0], len = n });
out("\n");
}
}
// ── Stub handlers (real logic lands in P3.5) ─────────────────────────
// Honest stubs: acknowledge the command and emit a known result.
handle_release_promote :: (p: *Parsed, json_mode: bool) {
ack("release promote", json_mode);
}
@@ -97,9 +130,8 @@ ack :: (cmd: string, json_mode: bool) {
out("\n");
}
// Route a parsed (group, command) to its stub handler. `parse` only
// returns a (group, command) present in the table, so one arm always
// matches.
// Route a parsed (group, command) to its handler. `parse` only returns a
// (group, command) present in the table, so one arm always matches.
dispatch :: (p: *Parsed, json_mode: bool) {
if p.group == "ci" and p.command == "publish" { handle_ci_publish(p, json_mode); return; }
if p.group == "release" and p.command == "promote" { handle_release_promote(p, json_mode); return; }
@@ -131,12 +163,17 @@ main :: () -> ! {
}
// Command table + flag specs live in this scope; `Parsed` holds VIEWS
// into them, used before `main` returns. Per-command flags arrive with
// the real handlers in P3.4/P3.5 — the global `--json` is recognized by
// the parser without being declared here.
// into them, used before `main` returns. `ci publish` requires
// `--manifest` + `--local-store`; the global `--json` is recognized by
// the parser without being declared here. promote/rollback flags arrive
// with their real handlers in P3.5.
no_flags : []FlagSpec = .[];
publish_flags : []FlagSpec = .[
FlagSpec.{ name = "manifest", takes_value = true, required = true },
FlagSpec.{ name = "local-store", takes_value = true, required = true },
];
cmds : []Command = .[
Command.{ group = "ci", command = "publish", flags = no_flags },
Command.{ group = "ci", command = "publish", flags = publish_flags },
Command.{ group = "release", command = "promote", flags = no_flags },
Command.{ group = "release", command = "rollback", flags = no_flags },
];

26
src/manifest/manifest.sx
View File

@@ -25,6 +25,10 @@
#import "modules/std.sx";
#import "modules/std/json.sx";
// Also reached through an alias so the json reader is called as `jsonp.parse`
// — a bare `parse` would bind to `std.cli`'s `parse` once both modules share
// one program (the `dist` CLI), which returns a different type.
jsonp :: #import "modules/std/json.sx";
#import "modules/fs.sx";
#import "../domain/platform.sx";
#import "../domain/validate.sx";
@@ -49,12 +53,20 @@ ManifestErr :: error {
// One artifact entry. `platform` + `path` are required; `filename`,
// `content_type`, and `metadata` are optional overrides that default to ""
// when absent. `metadata` is an opaque string mirroring Artifact.metadata.
//
// `size` / `sha256` are OPTIONAL declared expectations the publisher checks
// the on-disk file against: present -> that value IS the expectation;
// absent (`size == -1` / `sha256 == ""`) -> the publisher derives the
// expectation from the stored object, so a no-declaration manifest still
// validates trivially.
ManifestArtifact :: struct {
platform: Platform;
path: string;
filename: string;
content_type: string;
metadata: string;
size: s64 = -1;
sha256: string = "";
}
// A single publish: which app/version/channel, and the artifacts that make
@@ -107,6 +119,16 @@ opt_str :: (o: Object, key: string, alloc: Allocator) -> (string, !ManifestErr)
return dup_str(val.str, alloc);
}
// Optional integer field. Absent -> `default` (a documented default, not a
// silent one); present-but-not-an-integer -> WrongType.
opt_int :: (o: Object, key: string, default: s64) -> (s64, !ManifestErr) {
v := obj_find(o, key);
if v == null { return default; }
val := v!;
if val != .int_ { raise error.WrongType; }
return val.int_;
}
// Required array field. Absent -> MissingField; present-but-not-an-array ->
// WrongType.
req_arr :: (o: Object, key: string) -> (Array, !ManifestErr) {
@@ -141,6 +163,8 @@ artifact_from_json :: (o: Object, alloc: Allocator) -> (ManifestArtifact, !Manif
a.filename = try opt_str(o, "filename", alloc);
a.content_type = try opt_str(o, "content_type", alloc);
a.metadata = try opt_str(o, "metadata", alloc);
a.size = try opt_int(o, "size", -1);
a.sha256 = try opt_str(o, "sha256", alloc);
return a;
}
@@ -151,7 +175,7 @@ artifact_from_json :: (o: Object, alloc: Allocator) -> (ManifestArtifact, !Manif
// (WrongType), a missing/empty required field (MissingField), and an unknown
// artifact platform (UnknownPlatform). All strings are copied into `alloc`.
parse_manifest :: (src: string, alloc: Allocator) -> (Manifest, !ManifestErr) {
root, pe := parse(src, alloc);
root, pe := jsonp.parse(src, alloc);
if pe { raise error.BadJson; }
if root != .object { raise error.WrongType; }
ro := root.object;

313
src/publish/publish.sx Normal file
View File

@@ -0,0 +1,313 @@
// =====================================================================
// publish.sx — the local `dist ci publish` SUCCESS pipeline (subplan 03,
// Slice 1). Wires the prior modules into one end-to-end publish:
//
// manifest (P3.2) -> store (P2.2) -> common validation (P3.3) ->
// repository transaction + audit (P2.3) -> db.json persistence (P2.3)
//
// `run_publish(manifest_path, store_dir)` validates the manifest, finds or
// creates the app, drafts a release, content-addresses every artifact into
// `<store>/objects/<sha256>`, validates each stored file, commits the whole
// aggregate through the integrity-checked repo transaction (channel
// promotion included), records an audit event per upload / publish /
// promotion, persists `<store>/db.json`, and returns a `PublishOutcome` the
// CLI renders as stable JSON or a human summary.
//
// DECLARED-vs-DERIVED EXPECTATIONS (PO ruling): a manifest artifact may
// DECLARE `size` / `sha256`; when it does, that value is the expectation the
// common validation pass checks the on-disk file against. When it does NOT
// (size == -1 / sha256 == ""), the expectation is DERIVED from the stored
// object — the size read off disk and the sha256 returned by the store — so
// a no-declaration manifest validates trivially. (P3.4b will declare a wrong
// size/sha256 to drive the abort path; the repo transaction's rollback is
// left intact for it.)
//
// LOCAL DOWNLOAD URL FORM: `file://<abs-store>/objects/<sha256>`, where
// <abs-store> is the `--local-store` directory resolved to an absolute path
// (left as-is if already absolute, else joined onto the process cwd).
//
// ALLOCATION: everything the published `Repo` and the returned outcome hold
// is allocated from `context.allocator` (the process-lifetime default), so a
// one-shot publish never frees and never dangles.
// =====================================================================
#import "modules/std.sx";
#import "modules/std/json.sx";
#import "modules/fs.sx";
#import "../domain/platform.sx";
#import "../domain/app.sx";
#import "../domain/release.sx";
#import "../domain/artifact.sx";
#import "../domain/channel.sx";
#import "../domain/audit.sx";
#import "../store/store.sx";
#import "../repo/repo.sx";
#import "../validation/artifact_file.sx";
mani :: #import "../manifest/manifest.sx";
db :: #import "../repo/db.sx";
// libc handles the two facts the publish needs from the OS: the wall-clock
// time stamped onto the release / audit events, and the process cwd used to
// absolutize a relative `--local-store` into the download URL.
cstd :: #library "c";
c_time :: (tloc: *s64) -> s64 #foreign cstd "time";
c_getcwd :: (buf: [*]u8, size: usize) -> *u8 #foreign cstd "getcwd";
// Failure classes for the publish pipeline. One distinct tag per stage so
// the CLI (and P3.4b) can report precisely where a publish stopped.
// Manifest — the manifest failed to load / parse / validate.
// Store — an artifact's bytes could not be content-addressed.
// Validation — a stored artifact failed the common validation pass.
// Transaction — the repo's integrity-checked publish rejected the aggregate.
// Persist — db.json could not be written.
PublishError :: error {
Manifest,
Store,
Validation,
Transaction,
Persist,
}
// One artifact as the CLI reports it: its id, platform, size, content
// digest, and the local download URL its bytes live at.
PublishedArtifact :: struct {
id: string;
platform_name: string;
size_bytes: s64;
sha256: string;
url: string;
}
// The machine-readable result of a successful publish: the release identity
// and the artifacts that were stored under it.
PublishOutcome :: struct {
release_id: string;
app_id: string;
version: string;
channel: string;
artifacts: List(PublishedArtifact);
}
// ── helpers ───────────────────────────────────────────────────────────
// Current wall-clock time as unix epoch seconds (time(2) via a local slot,
// so no null pointer is needed).
now_secs :: () -> s64 {
t : s64 = 0;
return c_time(@t);
}
// Resolve `dir` to an absolute path: returned unchanged when already
// absolute, else joined onto the process cwd. Falls back to `dir` if the
// cwd can't be read.
abs_store :: (dir: string) -> string {
if dir.len > 0 and dir[0] == 47 { return dir; } // 47='/' already absolute
buf : [4096]u8 = ---;
r := c_getcwd(@buf[0], 4096);
if cast(s64) r == 0 { return dir; }
n := 0;
while buf[n] != 0 { n += 1; }
cwd := string.{ ptr = @buf[0], len = n };
return path_join(cwd, dir);
}
// The content type to record when the manifest doesn't declare one — the
// canonical type for the platform's artifact form (each is on the common
// validation allow-list).
default_content_type :: (p: Platform) -> string {
if p == .ios { return "application/octet-stream"; }
if p == .android_apk { return "application/vnd.android.package-archive"; }
if p == .macos { return "application/x-apple-diskimage"; }
if p == .linux { return "application/octet-stream"; }
return "application/x-msdownload"; // windows
}
// ── pipeline ──────────────────────────────────────────────────────────
run_publish :: (manifest_path: string, store_dir: string) -> (PublishOutcome, !PublishError) {
alloc := context.allocator;
// 1. Validate the manifest (parse + on-disk artifact existence).
m, me := mani.load_manifest(manifest_path, alloc);
if me { raise error.Manifest; }
base_dir := dirname(manifest_path);
abs := abs_store(store_dir);
now := now_secs();
repo := Repo.init();
st := Store.init(store_dir);
// 2. Find or create the app (keyed by slug).
slug := m.app;
app_id := "";
existing := repo.find_app_by_slug(slug);
if existing == null {
app_id = concat("app-", slug);
repo.create_app(App.{
id = app_id, slug = slug, display_name = slug,
owner = "ci", visibility = .private,
created_at = now, updated_at = now,
});
} else {
app_id = existing!.id;
}
// 3. Draft the release for this version/channel.
release_id := concat(concat(concat("rel-", slug), "-"), m.version);
rel := Release.{
id = release_id, app_id = app_id, version = m.version, build = 1,
channel = m.channel, notes = "", created_by = "ci",
created_at = now, published_at = now,
};
// 4. Per artifact: store by digest -> validate -> build the entity.
arts : List(Artifact) = .{};
out_arts : List(PublishedArtifact) = .{};
i := 0;
while i < m.artifacts.len {
ma := m.artifacts.items[i];
src := path_join(base_dir, ma.path);
// Content-address the bytes in memory via `put_bytes`. The streaming
// `put_file` path hashes through `hash.sha256_file`, which the sx
// LLVM backend miscompiles (codegen crash under `sx build`); reading
// the bytes and hashing them in memory yields the identical
// `objects/<sha256>` key with no streaming hash.
sb := read_file(src);
if sb == null { raise error.Store; }
bytes := sb!;
actual_size := bytes.len;
key, se := st.put_bytes(bytes);
if se { raise error.Store; }
// Declared expectation when present; derived from the stored object
// otherwise (so a no-declaration manifest validates trivially).
exp_size := if ma.size >= 0 then ma.size else actual_size;
exp_sha := if ma.sha256.len > 0 then ma.sha256 else key;
ct := if ma.content_type.len > 0 then ma.content_type else default_content_type(ma.platform);
outcome := validate_artifact_file(src, exp_size, exp_sha, ma.platform, ct);
if outcome.status != .valid { raise error.Validation; }
fname := if ma.filename.len > 0 then ma.filename else basename(src);
pname := db.platform_str(ma.platform);
art := Artifact.{
id = concat(concat(release_id, "-"), pname),
app_id = app_id, release_id = release_id,
platform = ma.platform, filename = fname, content_type = ct,
size_bytes = actual_size, sha256 = key, storage_key = key,
metadata = ma.metadata, validation_status = .valid,
};
arts.append(art, alloc);
url := concat(concat(concat("file://", abs), "/objects/"), key);
out_arts.append(PublishedArtifact.{
id = art.id, platform_name = pname, size_bytes = actual_size,
sha256 = key, url = url,
}, alloc);
i += 1;
}
// 5. Publish via the integrity-checked transaction (channel promotion
// included). Rollback on failure is left intact for P3.4b.
chan := Channel.{
app_id = app_id, name = m.channel, current_release_id = "",
policy = .manual, rollout_percent = 100,
};
pe := false;
repo.publish(rel, @arts, chan) catch { pe = true; };
if pe { raise error.Transaction; }
// Audit trail — only after a committed publish: one upload event per
// artifact, one publish event, one channel-promotion event.
j := 0;
while j < arts.len {
aid := arts.items[j].id;
repo.create_audit_event(AuditEvent.{
id = concat("evt-upload-", aid), actor = "ci",
action = "artifact.upload", target_type = "artifact",
target_id = aid, metadata = "", created_at = now,
});
j += 1;
}
repo.create_audit_event(AuditEvent.{
id = concat("evt-publish-", release_id), actor = "ci",
action = "release.publish", target_type = "release",
target_id = release_id, metadata = "", created_at = now,
});
repo.create_audit_event(AuditEvent.{
id = concat(concat(concat("evt-promote-", app_id), "-"), m.channel),
actor = "ci", action = "channel.promote", target_type = "channel",
target_id = m.channel, metadata = "", created_at = now,
});
// 6. Persist the whole model under the store.
persist_err := false;
db.save(repo, store_dir) catch { persist_err = true; };
if persist_err { raise error.Persist; }
return PublishOutcome.{
release_id = release_id, app_id = app_id,
version = m.version, channel = m.channel,
artifacts = out_arts,
};
}
// ── rendering ─────────────────────────────────────────────────────────
// Serialize a successful publish as one stable JSON object into the caller
// buffer `dst`, returning the bytes written. Member order is fixed (status,
// release{id,app_id,version,channel}, artifacts[]{id,platform,size_bytes,
// sha256,url}) by `std.json`'s insertion-order guarantee. Overflow surfaces
// on the error channel.
write_json :: (o: *PublishOutcome, dst: []u8) -> (s64, !JsonError) {
gpa := GPA.init();
root : Object = .{};
root.put("status", .str("published"), xx gpa);
rel : Object = .{};
rel.put("id", .str(o.release_id), xx gpa);
rel.put("app_id", .str(o.app_id), xx gpa);
rel.put("version", .str(o.version), xx gpa);
rel.put("channel", .str(o.channel), xx gpa);
root.put("release", .object(rel), xx gpa);
arts : Array = .{};
i := 0;
while i < o.artifacts.len {
a := o.artifacts.items[i];
ao : Object = .{};
ao.put("id", .str(a.id), xx gpa);
ao.put("platform", .str(a.platform_name), xx gpa);
ao.put("size_bytes", .int_(a.size_bytes), xx gpa);
ao.put("sha256", .str(a.sha256), xx gpa);
ao.put("url", .str(a.url), xx gpa);
arts.add(.object(ao), xx gpa);
i += 1;
}
root.put("artifacts", .array(arts), xx gpa);
rootv : Value = .object(root);
n := try write_to_buffer(rootv, dst);
return n;
}
// A readable one-publish summary for non-json mode: the release line plus
// one indented line per artifact (platform, digest, download URL).
human_summary :: (o: *PublishOutcome) -> string {
s := concat("published release ", o.release_id);
s = concat(s, concat(" (", concat(o.app_id, concat(" ", concat(o.version, ")")))));
s = concat(s, concat(" on channel ", concat(o.channel, "\n")));
i := 0;
while i < o.artifacts.len {
a := o.artifacts.items[i];
s = concat(s, concat(" ", concat(a.platform_name, concat(" ", concat(a.sha256, concat(" ", concat(a.url, "\n")))))));
i += 1;
}
return s;
}

6
src/repo/db.sx
View File

@@ -21,6 +21,10 @@
#import "modules/std.sx";
#import "modules/std/json.sx";
// Also reached through an alias so the json reader is called as `jsonp.parse`
// — a bare `parse` would bind to `std.cli`'s `parse` once both modules share
// one program (the `dist` CLI), which returns a different type.
jsonp :: #import "modules/std/json.sx";
#import "modules/fs.sx";
#import "../domain/platform.sx";
#import "../domain/app.sx";
@@ -360,7 +364,7 @@ audit_from_json :: (o: Object, alloc: Allocator) -> (AuditEvent, !LoadErr) {
load_into :: (repo: *Repo, bytes: string, scratch: Allocator) -> !LoadErr {
oa := repo.own_allocator;
root_val, pe := parse(bytes, scratch);
root_val, pe := jsonp.parse(bytes, scratch);
if pe { raise error.Parse; }
ro := try req_obj(root_val);