← back to the zoo

22. The zoo checks itself

Every self-running exhibit is rechecked on a schedule, and the front page says when.

The zoo is an ordinary app: a five-minute job re-runs its exhibits and stores the results, and root's hourly check of the platform arrives in RC_FACTS/platform.json.

This is the code serving the zoo right now: read from disk for this request, from commit 77ff74344d (staging). The zoo's own self-check fetches this page and compares it byte for byte with the file it runs.

1<?php
2// Exhibit 22. bin/selfcheck.php runs every five minutes (ratcloud.conf) and re-runs every
3// self-running exhibit's check, storing each result in zoo.db. The lights on this page are that
4// stored data, never live requests. It also reads root's own hourly check of the back of house,
5// which root publishes into RC_FACTS/platform.json.
6function back_of_house(): ?array {
7    $c = fact("platform.json")["check"] ?? null;
8    if (!$c) return null;
9    // Lines marked "(exhibit N)" are root's view of an exhibit that has its own card.
10    return ["at" => $c["at"], "lines" => array_values(array_filter($c["checks"], fn($l) => !str_starts_with($l["line"], "(exhibit")))];
11}
12
13return [
14    "n" => 22, "wing" => "Operations",
15    "title" => "The zoo checks itself",
16    "promise" => "Every self-running exhibit is rechecked on a schedule, and the front page says when.",
17    "block" => "The zoo is an ordinary app: a five-minute job re-runs its exhibits and stores the results, and root's hourly check of the platform arrives in RC_FACTS/platform.json.",
18    "files" => ["bin/selfcheck.php"],
19    "show" => function (?array $me): string {
20        $t = meta("checked_at");
21        $out = "<p>Checked <b>" . ago($t ? (int)$t : null) . "</b>" . (($d = meta("checked_ms")) ? " (took " . round($d / 1000, 1) . " s)" : "") . ".</p>";
22        $b = back_of_house();
23        if (!$b) return $out . '<p class="muted">Root\'s check of the back of house is not published to apps yet (RC_FACTS/platform.json is missing).</p>';
24        $out .= "<p>The back of house, checked by root " . ago(strtotime($b["at"])) . ":</p><ul class=\"list\">";
25        foreach ($b["lines"] as $l) $out .= '<li><span class="tag ' . ($l["status"] === "pass" ? "ok" : "bad") . '">' . h($l["status"]) . '</span><span>' . h($l["line"]) . ' <small class="muted">' . h($l["detail"]) . "</small></span></li>";
26        return $out . "</ul>";
27    },
28    // Runs last in bin/selfcheck.php, after every other exhibit.
29    "check" => function (): array {
30        $red = [];
31        foreach (exhibits() as $n => $e) if ($n !== 22 && $e["kind"] === "self" && light($e)["color"] === "red") $red[] = $n;
32        $b = back_of_house();
33        if (!$b) return [false, "root's back-of-house check is not readable: RC_FACTS/platform.json is missing"];
34        if (time() - strtotime($b["at"]) > 3 * 3600) return [false, "root's back-of-house check is " . intdiv(time() - strtotime($b["at"]), 3600) . " hours old"];
35        $fail = array_filter($b["lines"], fn($l) => $l["status"] !== "pass");
36        if ($fail) return [false, "back of house: " . implode("; ", array_map(fn($l) => $l["line"], $fail))];
37        if ($red) return [false, "red: exhibit" . (count($red) > 1 ? "s " : " ") . implode(", ", $red)];
38        return [true, "every self-running exhibit is green, and root's " . count($b["lines"]) . " back-of-house lines pass"];
39    },
40];

1<?php
2// Exhibit 22's job: every five minutes (ratcloud.conf), run every self-running exhibit's check
3// and store the results as lights. Exhibit 22 runs last, since it judges the others.
4require __DIR__ . "/../lib/zoo.php";
5$t = microtime(true);
6$all = exhibits();
7$last = $all[22];
8unset($all[22]);
9foreach ([...$all, $last] as $e) {
10    if (!$e["check"] || $e["kind"] !== "self") continue;
11    $r = run_check($e);
12    if (!$r["ok"]) echo "exhibit {$r['n']}: {$r['detail']}\n";   // into the journal, so log.txt
13}
14require __DIR__ . "/../lib/alerts.php";
15try { alerts_run(); } catch (Throwable $x) { echo "alerts: " . $x->getMessage() . "\n"; }
16meta_set("checked_at", (string)time());
17meta_set("checked_ms", (string)(int)((microtime(true) - $t) * 1000));