visual check

.github/workflows/macos-build.yml

@@ -64,6 +64,20 @@ jobs:
         run: python3 packaging/macos/build.py --build-number '${{ github.run_number }}'
       - name: Verify relocated startup
         run: python3 packaging/macos/smoke.py dist/ZoiteChat.app
+      # Desktop capture can be unavailable on a runner image. Preserve the
+      # existing build gate and publish the separate visual-check outcome.
+      - name: Exercise IRC and capture the macOS application window
+        continue-on-error: true
+        timeout-minutes: 5
+        run: python3 packaging/macos/visual-smoke.py dist/ZoiteChat.app
+      - name: Upload screenshot and visual-check evidence
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
+        with:
+          name: ZoiteChat-macos-${{ matrix.arch }}-visual-check
+          path: dist/visual-check/
+          if-no-files-found: ignore
+          retention-days: 14
       - name: Create artifact preserving the application signature
         run: ditto -c -k --keepParent dist/ZoiteChat.app dist/ZoiteChat-${{ matrix.arch }}.zip
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7

packaging/macos/README.md

@@ -59,6 +59,42 @@ The smoke check copies the app to a path containing spaces, verifies its signatu
 runs `--version`, and keeps an isolated GUI process alive for ten seconds without
 connecting to IRC. It tests startup, not interaction or long-running stability.
 
+### Screenshots and a loopback IRC check without owning a Mac
+
+The ordinary ARM and Intel workflow jobs also run a separate visual check.
+After pushing this change to a pull request, open the new **macOS build** run
+and download `ZoiteChat-macos-arm64-visual-check` and
+`ZoiteChat-macos-x86_64-visual-check` from **Artifacts**. Each contains:
+
+- `zoitechat.png`: the actual running application's window on that architecture.
+- `result.json`: the checks completed, window dimensions, architecture and commit.
+- `irc.log`: the temporary loopback server's registration, channel and PING/PONG traffic.
+- `startup.log`: the application's startup diagnostics.
+
+The check copies the built app to a path containing spaces, verifies its
+signature, starts it with temporary settings, connects to a server bound only
+to `127.0.0.1`, joins `#ci-smoke`, receives sample text and verifies its PONG
+response. It captures the largest visible window owned by that app's process
+and rejects failed or nearly uniform captures. No public IRC server or account
+is used. The original application artifact is not modified.
+
+This is a supplemental check with `continue-on-error`: a green build alone
+does **not** mean the visual check passed. Inspect `result.json` for
+`"status": "passed"` and view the PNG. If the runner has no desktop or denies
+Screen Recording, the step reports failure and publishes diagnostics instead
+of claiming a screenshot. No signing secrets are needed. The check runs in
+the ordinary build job; it does not alter the release-signing job.
+
+It can also run on a Mac with a logged-in desktop:
+
+```sh
+python3 packaging/macos/visual-smoke.py dist/ZoiteChat.app --output dist/visual-check
+```
+
+Use a new output directory for each run. This verifies a narrow set of startup,
+IRC and rendering behavior; it does not test keyboard/clipboard interaction,
+TLS, notifications, Gatekeeper distribution, or long-running stability.
+
 The audit verifies the entire dependency closure, including GTK image/input
 modules, GIO TLS, Enchant providers, OpenSSL providers and ZoiteChat plugins. It
 fails on missing libraries, external symlinks/search paths, architecture mismatches

packaging/macos/capture-window.swift

@@ -0,0 +1,72 @@
+// Capture only the tested process's window; a desktop-only image is not evidence.
+import AppKit
+import CoreGraphics
+import Foundation
+
+func fail(_ message: String) -> Never {
+    FileHandle.standardError.write(Data((message + "\n").utf8))
+    exit(1)
+}
+
+guard CommandLine.arguments.count == 3,
+      let pid = Int32(CommandLine.arguments[1]) else {
+    fail("Usage: capture-window PID OUTPUT.png")
+}
+let output = URL(fileURLWithPath: CommandLine.arguments[2])
+let deadline = Date().addingTimeInterval(30)
+var chosen: [String: Any]?
+while Date() < deadline {
+    guard let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated else {
+        fail("ZoiteChat is not running")
+    }
+    app.activate(options: [])
+    RunLoop.current.run(until: Date().addingTimeInterval(0.2))
+    let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements],
+                                            kCGNullWindowID) as? [[String: Any]] ?? []
+    let owned = windows.filter { window in
+        guard (window[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid,
+              (window[kCGWindowLayer as String] as? NSNumber)?.intValue == 0,
+              let bounds = window[kCGWindowBounds as String] as? [String: Any],
+              let width = bounds["Width"] as? NSNumber,
+              let height = bounds["Height"] as? NSNumber else { return false }
+        return width.doubleValue >= 600 && height.doubleValue >= 300
+    }
+    chosen = owned.max { left, right in
+        func area(_ window: [String: Any]) -> Double {
+            let bounds = window[kCGWindowBounds as String] as? [String: NSNumber] ?? [:]
+            return (bounds["Width"]?.doubleValue ?? 0) * (bounds["Height"]?.doubleValue ?? 0)
+        }
+        return area(left) < area(right)
+    }
+    if chosen != nil { break }
+}
+guard let window = chosen,
+      let identifier = window[kCGWindowNumber as String] as? NSNumber else {
+    fail("No visible ZoiteChat window found in this runner's desktop session")
+}
+let capture = Process()
+capture.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
+capture.arguments = ["-x", "-o", "-t", "png", "-l", identifier.stringValue, output.path]
+do { try capture.run() } catch { fail("Cannot start screencapture: \(error)") }
+capture.waitUntilExit()
+guard capture.terminationStatus == 0,
+      let png = try? Data(contentsOf: output),
+      let bitmap = NSBitmapImageRep(data: png),
+      bitmap.pixelsWide >= 600, bitmap.pixelsHigh >= 300 else {
+    fail("Application window capture failed; check runner Screen Recording permissions")
+}
+var colors = Set<String>()
+for y in stride(from: 0, to: bitmap.pixelsHigh, by: max(1, bitmap.pixelsHigh / 60)) {
+    for x in stride(from: 0, to: bitmap.pixelsWide, by: max(1, bitmap.pixelsWide / 80)) {
+        if let color = bitmap.colorAt(x: x, y: y)?.usingColorSpace(.deviceRGB) {
+            colors.insert("\(Int(color.redComponent * 255)),\(Int(color.greenComponent * 255)),\(Int(color.blueComponent * 255))")
+        }
+    }
+}
+guard colors.count >= 8 else { fail("Window capture is blank or nearly uniform") }
+let evidence: [String: Any] = ["pid": pid, "window_id": identifier.intValue,
+    "width": bitmap.pixelsWide, "height": bitmap.pixelsHigh, "sampled_colors": colors.count]
+do {
+    let data = try JSONSerialization.data(withJSONObject: evidence, options: [.sortedKeys])
+    print(String(decoding: data, as: UTF8.self))
+} catch { fail("Cannot serialize screenshot evidence: \(error)") }

packaging/macos/tests/test_visual_smoke.py

@@ -0,0 +1,85 @@
+"""Check that visual-test evidence requires a real IRC handshake and reply."""
+import importlib.util
+from pathlib import Path
+import socket
+import threading
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+    "visual_smoke", Path(__file__).resolve().parents[1] / "visual-smoke.py")
+visual = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(visual)
+
+
+class VisualSmokeTests(unittest.TestCase):
+    def test_tls_clienthello_is_rejected_without_waiting_for_a_newline(self):
+        with visual.IRCServer() as server:
+            thread = threading.Thread(target=server.serve_forever, daemon=True)
+            thread.start()
+            try:
+                with socket.create_connection(server.server_address, timeout=3) as client:
+                    # A TLS handshake record, with no IRC line terminator.
+                    client.sendall(b"\x16\x03\x03\x00\x00")
+                    self.assertEqual(client.recv(1), b"")
+                self.assertIn("Received TLS", server.error)
+                self.assertFalse(server.joined.is_set())
+                self.assertFalse(server.pong.is_set())
+            finally:
+                server.shutdown()
+                thread.join(timeout=3)
+
+    def test_registration_join_and_challenge_reply(self):
+        with visual.IRCServer() as server:
+            thread = threading.Thread(target=server.serve_forever, daemon=True)
+            thread.start()
+            try:
+                with socket.create_connection(server.server_address, timeout=3) as client:
+                    stream = client.makefile("rb")
+
+                    def until(marker):
+                        lines = []
+                        for _ in range(30):
+                            line = stream.readline().decode("utf-8").rstrip()
+                            self.assertTrue(line, "server closed before expected response")
+                            lines.append(line)
+                            if marker in line:
+                                return lines
+                        self.fail("expected response missing: " + marker)
+
+                    client.sendall(b"CAP LS 302\r\nNICK MacTester\r\nUSER ci 0 * :CI\r\n")
+                    until(" CAP * LS :")
+                    self.assertFalse(server.joined.is_set())
+                    client.sendall(b"CAP END\r\n")
+                    until(" 376 MacTester ")
+                    client.sendall(b"JOIN #ci-smoke\r\n")
+                    messages = until("PING :" + server.token)
+                    self.assertTrue(any("PRIVMSG #ci-smoke" in line for line in messages))
+                    self.assertTrue(server.joined.wait(timeout=1))
+                    self.assertFalse(server.pong.is_set())
+                    client.sendall(b"PONG :wrong-token\r\nPING :barrier\r\n")
+                    until("PONG :barrier")
+                    self.assertFalse(server.pong.is_set())
+                    client.sendall(("PONG :" + server.token + "\r\n").encode())
+                    self.assertTrue(server.pong.wait(timeout=1))
+                    client.sendall(b"QUIT\r\n")
+                    stream.close()
+            finally:
+                server.shutdown()
+                thread.join(timeout=3)
+
+    def test_early_disconnect_is_not_success(self):
+        with visual.IRCServer() as server:
+            thread = threading.Thread(target=server.serve_forever, daemon=True)
+            thread.start()
+            try:
+                with socket.create_connection(server.server_address, timeout=3) as client:
+                    client.sendall(b"QUIT\r\n")
+                self.assertFalse(server.joined.is_set())
+                self.assertFalse(server.pong.is_set())
+            finally:
+                server.shutdown()
+                thread.join(timeout=3)
+
+
+if __name__ == "__main__":
+    unittest.main()

packaging/macos/visual-smoke.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""Exercise a relocated Mac bundle against loopback IRC and capture its window."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+import platform
+import secrets
+import shutil
+import socketserver
+import subprocess
+import tempfile
+import threading
+import time
+
+
+class IRCServer(socketserver.ThreadingTCPServer):
+    daemon_threads = True
+
+    def __init__(self):
+        self.joined = threading.Event()
+        self.pong = threading.Event()
+        self.error: str | None = None
+        self.token = secrets.token_hex(12)
+        self.transcript: list[str] = []
+        super().__init__(("127.0.0.1", 0), IRCHandler)
+
+
+class IRCHandler(socketserver.StreamRequestHandler):
+    def handle(self):
+        self.connection.settimeout(60)
+        nick, user, registered, negotiating = "", False, False, False
+
+        def send(line):
+            self.server.transcript.append("S: " + line)
+            self.wfile.write((line + "\r\n").encode("utf-8"))
+            self.wfile.flush()
+
+        try:
+            # A TLS ClientHello is binary and need not contain a newline. Do
+            # not leave the fixture waiting for an IRC line until its timeout.
+            if self.rfile.peek(1).startswith(b"\x16"):
+                self.server.error = "Received TLS on the plaintext loopback IRC fixture; launch with SERVER -insecure"
+                self.server.transcript.append("Error: " + self.server.error)
+                return
+            while raw := self.rfile.readline(8192):
+                line = raw.decode("utf-8", errors="replace").rstrip("\r\n")
+                self.server.transcript.append("C: " + line)
+                parts = line.split()
+                if not parts:
+                    continue
+                command = parts[0].upper()
+                if command == "CAP" and len(parts) > 1:
+                    if parts[1].upper() == "LS":
+                        negotiating = True
+                        send(":ci.local CAP * LS :")
+                    elif parts[1].upper() == "END":
+                        negotiating = False
+                    elif parts[1].upper() == "REQ":
+                        send(":ci.local CAP * NAK :" + line.partition(" :")[2])
+                elif command == "NICK" and len(parts) > 1:
+                    nick = parts[1].lstrip(":")
+                elif command == "USER":
+                    user = True
+                elif command == "PING" and len(parts) > 1:
+                    send("PONG :" + parts[-1].lstrip(":"))
+                elif command == "PONG" and parts[-1].lstrip(":") == self.server.token:
+                    self.server.pong.set()
+                elif command == "JOIN" and registered:
+                    channel = parts[1].lstrip(":").split(",")[0]
+                    send(f":{nick}!ci@localhost JOIN :{channel}")
+                    send(f":ci.local 332 {nick} {channel} :ZoiteChat native macOS visual check")
+                    send(f":ci.local 353 {nick} = {channel} :@CiGuide {nick}")
+                    send(f":ci.local 366 {nick} {channel} :End of names")
+                    for message in (
+                        "ZoiteChat is running on macOS " + platform.machine() + ".",
+                        "Connected to the temporary IRC server and joined this channel.",
+                        "Unicode rendering: café — 日本語 — 👋",
+                        "This screenshot is captured from the running application window.",
+                    ):
+                        send(f":CiGuide!ci@localhost PRIVMSG {channel} :{message}")
+                    send("PING :" + self.server.token)
+                    self.server.joined.set()
+                elif command == "QUIT":
+                    return
+                if nick and user and not registered and not negotiating:
+                    registered = True
+                    send(f":ci.local 001 {nick} :Welcome to the ZoiteChat CI test server")
+                    send(f":ci.local 005 {nick} CHANTYPES=# PREFIX=(ov)@+ NETWORK=ZoiteChat-CI :are supported")
+                    send(f":ci.local 376 {nick} :End of MOTD")
+        except (OSError, ValueError) as error:
+            self.server.transcript.append("Connection closed: " + str(error))
+
+
+def stop(process):
+    if process.poll() is None:
+        process.terminate()
+        try:
+            process.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            process.wait(timeout=5)
+
+
+def check(app: Path, output: Path):
+    report = {"status": "failed", "architecture": platform.machine(),
+              "source_commit": os.environ.get("GITHUB_SHA"),
+              "checks": [], "limitations": "Loopback IRC and window capture; no TLS, input, notifications or stability coverage."}
+    with IRCServer() as server, tempfile.TemporaryDirectory(prefix="ZoiteChat visual check ") as temporary:
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            directory = Path(temporary)
+            capture = directory / "capture-window"
+            subprocess.run(["xcrun", "swiftc", str(Path(__file__).with_name("capture-window.swift")),
+                            "-o", str(capture)], check=True, timeout=120)
+            relocated = directory / "Relocated App/ZoiteChat.app"
+            shutil.copytree(app.resolve(), relocated)
+            subprocess.run(["codesign", "--verify", "--deep", "--strict", str(relocated)], check=True, timeout=30)
+            report["checks"].append("relocated bundle signature verified")
+            config = directory / "config"
+            config.mkdir()
+            (config / "zoitechat.conf").write_text(
+                "gui_slist_skip = 1\ngui_win_width = 1100\ngui_win_height = 700\n"
+                "gui_win_left = 60\ngui_win_top = 60\n"
+                "irc_nick1 = MacTester\nirc_user_name = MacTester\nirc_real_name = Visual check\n"
+                "net_proxy_type = 0\nidentd_server = 0\n", encoding="utf-8")
+            binary = relocated / "Contents/MacOS/zoitechat"
+            with (output / "startup.log").open("w", encoding="utf-8") as log:
+                # SERVER defaults to TLS even for an irc:// URL in this branch.
+                # Explicit plaintext is restricted to our loopback-only fixture.
+                command = f"SERVER -insecure -noproxy irc://127.0.0.1:{server.server_address[1]}/ci-smoke"
+                process = subprocess.Popen([str(binary), "--no-auto", "--cfgdir", str(config),
+                                            "--command", command],
+                                           stdout=log, stderr=log)
+                try:
+                    deadline = time.monotonic() + 60
+                    while not (server.joined.is_set() and server.pong.is_set()):
+                        if server.error:
+                            raise RuntimeError(server.error)
+                        if process.poll() is not None:
+                            raise RuntimeError(f"Application exited with code {process.returncode}; see startup.log")
+                        if time.monotonic() >= deadline:
+                            raise RuntimeError("Timed out waiting for channel join and PING/PONG; see irc.log")
+                        time.sleep(0.2)
+                    report["checks"] += ["connected to loopback IRC", "joined channel", "answered server PING"]
+                    time.sleep(1)
+                    window = subprocess.check_output([str(capture), str(process.pid), str(output / "zoitechat.png")],
+                                                     text=True, timeout=40)
+                    report["window"] = json.loads(window)
+                    if process.poll() is not None:
+                        raise RuntimeError("Application exited during capture")
+                    report["checks"].append("captured a nonblank application window")
+                    report["status"] = "passed"
+                finally:
+                    stop(process)
+        except Exception as error:
+            report["error"] = str(error)
+            raise
+        finally:
+            server.shutdown()
+            thread.join(timeout=5)
+            (output / "irc.log").write_text("\n".join(server.transcript) + "\n", encoding="utf-8")
+            (output / "result.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("app", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("dist/visual-check"))
+    args = parser.parse_args()
+    if platform.system() != "Darwin":
+        parser.error("This check must run on macOS with a logged-in desktop")
+    args.output.mkdir(parents=True, exist_ok=False)
+    check(args.app, args.output.resolve())
+    print("Visual check passed; screenshot and evidence saved to " + str(args.output))
+
+
+if __name__ == "__main__":
+    main()