← back to the zoo

13. Apps can save files for you

An app can save a file into your storage, under its own folder, without ever holding storage's keys.

rc_app_headers("storage", true) proves to storage which app calls and for whom; storage saves only under Apps / <app>. In bulk: app/begin gives presigned PUTs (200 a call), the bytes go straight to B2, app/commit checks them all at once, app/urls hands out downloads.

This is the code serving the zoo right now: read from disk for this request, from commit 77ff74344d (staging). The zoo's own self-check fetches this page and compares it byte for byte with the file it runs.

1<?php
2// Exhibit 13. The zoo holds no storage keys. It saves files into a person's storage by showing
3// storage an assertion from account that says "zoo, for this person" (rc_app_headers("storage",
4// true)), in bulk: app/begin hands out presigned PUTs, the bytes go straight to B2, app/commit
5// asks B2 once whether they all arrived, app/urls hands out presigned GETs (lib/bulk.php). Storage
6// decides the folder (Apps / Zoo); the app cannot choose another.
7require_once ZOO_ROOT . "/lib/bulk.php";
8
9function postcard_png(array $me, string $line = ""): string {
10    // Every line fits inside the border: GD's built-in fonts are fixed-width (font 2: 6 px,
11    // font 5: 9 px), so the widest line, the 20-character date, is 120 px in a 148 px frame.
12    $im = imagecreatetruecolor(160, 104);
13    $id = (int)$me["id"];
14    imagefill($im, 0, 0, imagecolorallocate($im, 40 + ($id * 37) % 120, 90 + ($id * 61) % 100, 120 + ($id * 17) % 100));
15    $w = imagecolorallocate($im, 255, 255, 250);
16    imagerectangle($im, 5, 5, 154, 98, $w);
17    imagestring($im, 3, 14, 16, "Greetings from", $w);
18    imagestring($im, 5, 14, 38, "the zoo", $w);
19    imagestring($im, 2, 14, 74, $line !== "" ? $line : gmdate("Y-m-d H:i") . " UTC", $w);
20    ob_start(); imagepng($im); return ob_get_clean();
21}
22
23// Save $cards (path => PNG bytes) for the signed-in person and read every one back:
24// begin all, PUT all to B2, commit all, ask for all their URLs, GET them, compare.
25function save_cards(array $cards): array {
26    $paths = array_keys($cards);
27    [$s, $r] = storage_call("begin", ["files" => array_map(fn($p) => ["path" => $p, "size" => strlen($cards[$p]), "type" => "image/png"], $paths)]);
28    $beg = $r["files"] ?? [];
29    if ($s !== 200 || count(array_filter($beg, fn($f) => isset($f["url"]))) !== count($cards)) return ["error" => "storage would not begin them ($s): " . json_encode($r["error"] ?? $beg[0] ?? $r), "status" => 502];
30    $puts = put_many(array_map(fn($f, $p) => [$f["url"], $cards[$p], $f["type"]], $beg, $paths));
31    if (count(array_filter($puts, fn($c) => $c === 200)) !== count($cards)) return ["error" => "B2 refused some PUTs: " . json_encode(array_count_values($puts)), "status" => 502];
32    [$s, $r] = storage_call("commit", ["ids" => array_column($beg, "id")]);
33    $files = array_column(array_filter($r["files"] ?? [], fn($f) => isset($f["file"])), "file");
34    if ($s !== 200 || count($files) !== count($cards)) return ["error" => "storage would not commit them all ($s): " . json_encode($r["error"] ?? $r["files"][0] ?? $r), "status" => 502];
35    [$s, $r] = storage_call("urls", ["ids" => array_column($files, "id")]);
36    $urls = array_column($r["files"] ?? [], "url");
37    if ($s !== 200 || count($urls) !== count($cards)) return ["error" => "storage would not hand out their URLs ($s)", "status" => 502];
38    $back = get_many($urls);
39    $matched = 0;
40    foreach ($paths as $i => $p) if (($back[$i][0] ?? 0) === 200 && hash_equals(hash("sha256", $cards[$p]), hash("sha256", $back[$i][1]))) $matched++;
41    return ["files" => $files, "matched" => $matched, "back" => $back];
42}
43
44function postcard(array $me): array {
45    $png = postcard_png($me);
46    $r = save_cards(["postcards/postcard.png" => $png]);
47    if (isset($r["error"])) return $r;
48    $f = $r["files"][0];
49    return ["file" => ["id" => $f["id"], "path" => "Apps/Zoo/$f[path]"], "bytes" => strlen($png), "matched" => $r["matched"] === 1,
50            "image" => "data:image/png;base64," . base64_encode($r["back"][0][1])];
51}
52
53// The check's batch: 50 postcards in one begin, one commit and one urls call, then listed.
54function postcards_50(array $me): array {
55    $t = microtime(true);
56    $cards = [];
57    for ($i = 1; $i <= 50; $i++) $cards[sprintf("postcards/check/%02d.png", $i)] = postcard_png($me, sprintf("no. %02d of 50", $i));
58    $r = save_cards($cards);
59    if (isset($r["error"])) return $r;
60    $listed = [];
61    for ($after = 0, $pages = 0; $after !== null && $pages < 20; $pages++) {
62        [$s, $l] = storage_call("list", ["after" => $after, "limit" => 1000]);
63        foreach ($l["files"] ?? [] as $f) $listed[$f["path"]] = true;
64        $after = $l["next"] ?? null;
65    }
66    $ids = array_column($r["files"], "id");
67    return ["count" => count($cards), "matched" => $r["matched"], "listed" => count(array_intersect_key($listed, $cards)),
68            "path" => "Apps/Zoo/postcards/check/", "ids" => [min($ids), max($ids)], "ms" => (int)((microtime(true) - $t) * 1000)];
69}
70
71return [
72    "n" => 13, "try" => "save a postcard", "wing" => "Storage", "kind" => "self",
73    "title" => "Apps can save files for you",
74    "promise" => "An app can save a file into your storage, under its own folder, without ever holding storage's keys.",
75    "block" => 'rc_app_headers("storage", true) proves to storage which app calls and for whom; storage saves only under Apps / <app>. In bulk: app/begin gives presigned PUTs (200 a call), the bytes go straight to B2, app/commit checks them all at once, app/urls hands out downloads.',
76    "files" => ["lib/bulk.php"],
77    "show" => function (?array $me): string {
78        $link = '<a href="' . h(rc_app_url("storage")) . '/" target="_blank" rel="noopener">your storage, under Apps / Zoo</a>';
79        $s = stored(13);
80        $last = $s && $s["ok"] ? '<p class="muted small">The self-check saves 50 postcards for the robot each time: one begin, 50 PUTs straight to B2, one commit, one call for their 50 URLs, '
81            . (int)($s["data"]["ms"] ?? 0) . ' ms in all, ' . ago((int)$s["at"]) . '.</p>' : "";
82        if (!$me) return '<p class="muted"><a href="' . h(rc_signin_url()) . '">Sign in</a> to have the zoo save a postcard for you.</p>' . $last;
83        return '<p><button id="postcard" class="primary">Save a postcard</button> <span class="out" id="postcard-out"></span></p><p id="postcard-img"></p>'
84             . '<p class="muted">It appears in ' . $link . '.</p>' . $last;
85    },
86    "api" => function (string $do, ?array $me, array $in, bool $post): ?array {
87        if (!in_array($do, ["postcard", "postcards"], true) || !$post) return null;
88        if (!$me) return ["error" => "sign in first", "status" => 401];
89        return $do === "postcard" ? postcard($me) : postcards_50($me);
90    },
91    // The robot has the zoo save 50 postcards into the robot's storage in one batch, read them all
92    // back and find them in the listing.
93    "check" => function (): array {
94        require_once ZOO_ROOT . "/lib/robot.php";
95        [$code, , $j] = robot_at_zoo()->post("https://" . env("RC_HOST") . "/api/13/postcards");
96        if ($code !== 200 || !is_array($j)) return [false, "saving 50 postcards failed ($code): " . (is_array($j) ? ($j["error"] ?? "") : substr((string)$j, 0, 120))];
97        if ($j["matched"] !== 50) return [false, "only {$j['matched']} of 50 postcards came back unchanged", $j];
98        if ($j["listed"] !== 50) return [false, "only {$j['listed']} of 50 postcards are in the zoo's listing", $j];
99        return [true, "saved 50 postcards under {$j['path']} for the robot (one begin, PUTs straight to B2, one commit, one call for 50 URLs), fetched all 50 back unchanged and found them listed, in {$j['ms']} ms", $j];
100    },
101    "script" => <<<'JS'
102document.getElementById("postcard")?.addEventListener("click", async (ev) => {
103  const out = document.getElementById("postcard-out");
104  ev.target.disabled = true; out.textContent = "saving…";
105  const j = await zoo.call("/api/13/postcard", {});
106  ev.target.disabled = false;
107  if (j.error) { out.textContent = j.error; return; }
108  out.textContent = "saved to " + j.file.path + " (file " + j.file.id + ", " + j.bytes + " bytes)" + (j.matched ? ", fetched back unchanged" : ", but the copy differs");
109  document.getElementById("postcard-img").innerHTML = '<img alt="your postcard" width="160" height="104" src="' + zoo.esc(j.image) + '">';
110});
111
112JS,
113];

1<?php
2// Storage in bulk, the way any app calls it (storage's README, "For an app: in bulk"): a JSON call
3// to storage with rc_app_headers("storage", true), and the bytes straight to B2 and back on the
4// presigned URLs it hands out. No storage key, and no byte passes through storage's PHP.
5//
6//     [$s, $r] = storage_call("begin", ["files" => [["path" => "postcards/a.png", "size" => 812, "type" => "image/png"]]]);
7//     // $r["files"][0]: {id, url, type, size, expires}; PUT the bytes to url with that Content-Type
8//     storage_call("commit", ["ids" => [$id, ...]]);       // one B2 listing for the whole begin
9//     storage_call("urls", ["ids" => [$id, ...]]);         // presigned GETs, five minutes each
10//     storage_call("list", ["after" => 0, "limit" => 1000]);   // what the zoo saved for this person
11
12function storage_call(string $op, array $in = []): array {
13    [$s, $out] = call_app("storage", "/app/$op", true, "POST", json_encode($in), ["Content-Type: application/json"], 55);
14    $j = json_decode($out, true);
15    return [$s, is_array($j) ? $j : ["error" => "storage answered $s: " . substr($out, 0, 160)]];
16}
17
18// PUT many bodies to their presigned URLs, 16 at a time: [status, ...] in order.
19// $jobs: [[url, bytes, content type], ...]
20function put_many(array $jobs): array {
21    return many(array_map(function ($j) {
22        $c = curl_init($j[0]);
23        curl_setopt_array($c, [CURLOPT_CUSTOMREQUEST => "PUT", CURLOPT_POSTFIELDS => $j[1], CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 30,
24            CURLOPT_HTTPHEADER => ["Content-Type: $j[2]", "Expect:"]]);
25        return $c;
26    }, $jobs));
27}
28
29// GET many presigned URLs: [[status, body], ...] in order.
30function get_many(array $urls): array {
31    return many(array_map(function ($u) {
32        $c = curl_init($u);
33        curl_setopt_array($c, [CURLOPT_RETURNTRANSFER => true, CURLOPT_TIMEOUT => 30]);
34        return $c;
35    }, $urls), true);
36}
37
38function many(array $handles, bool $bodies = false): array {
39    $mh = curl_multi_init();
40    $out = [];
41    foreach (array_chunk($handles, 16, true) as $chunk) {
42        foreach ($chunk as $c) curl_multi_add_handle($mh, $c);
43        do { curl_multi_exec($mh, $running); curl_multi_select($mh, 0.2); } while ($running);
44        foreach ($chunk as $i => $c) {
45            $code = (int)curl_getinfo($c, CURLINFO_HTTP_CODE);
46            $out[$i] = $bodies ? [$code, (string)curl_multi_getcontent($c)] : $code;
47            curl_multi_remove_handle($mh, $c);
48        }
49    }
50    return $out;
51}