feat: publish downloadable wakeword trainer source

2026-08-14 05:03:00 -05:00
parent 000671eb93
commit aab9ce0fe7
30 changed files with 6757 additions and 2 deletions
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Start one sealed offline Wakeword Training attempt.
+
+The command is deliberately operator-only.  It has no gateway/API-key path and
+refuses to start unless both ``--enable`` and an active maintenance window are
+explicitly supplied.  See docs/wakeword-training-operator-runner.md.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+from pathlib import Path
+
+from wakeword_training import (
+    DockerTrainingExecutor,
+    FakeTrainingExecutor,
+    NvidiaSmiGpuEvidenceSampler,
+    NvidiaSmiProcessInspector,
+    RunnerSettings,
+    StaticGpuProcessInspector,
+    StaticMaintenanceWindow,
+    WakewordTrainingRequest,
+    WakewordTrainingRunner,
+    terminal_exit_code,
+)
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--request", required=True, type=Path, help="immutable request JSON")
+    parser.add_argument("--work-root", required=True, type=Path, help="operator run root")
+    parser.add_argument("--enable", action="store_true", help="explicitly permit this invocation")
+    parser.add_argument(
+        "--maintenance-window-active",
+        action="store_true",
+        help="attest that the configured GPU maintenance window is active",
+    )
+    parser.add_argument("--retry-of", help="terminal matching run ID for a new immutable attempt")
+    parser.add_argument("--timeout-s", type=float, default=6 * 60 * 60)
+    parser.add_argument(
+        "--fake", action="store_true", help="CI/local quarantined contract executor"
+    )
+    parser.add_argument(
+        "--image-ref", help="digest-pinned training image, e.g. registry/image@sha256:…"
+    )
+    parser.add_argument("--gpu-uuid", help="stable configured RTX UUID, never a mutable ordinal")
+    parser.add_argument(
+        "--approved-inputs-dir",
+        type=Path,
+        help="hash-locked data mount; mounted read-only at /inputs",
+    )
+    parser.add_argument(
+        "--allow-active-gpu-processes",
+        action="store_true",
+        help="explicit shared-GPU override; never terminates another process",
+    )
+    parser.add_argument(
+        "--stream-output", action="store_true", help="mirror container progress to stderr"
+    )
+    return parser
+
+
+async def _main(args: argparse.Namespace) -> int:
+    request = WakewordTrainingRequest.from_file(args.request)
+    if args.fake:
+        executor = FakeTrainingExecutor()
+    else:
+        if not (args.image_ref and args.gpu_uuid and args.approved_inputs_dir):
+            raise SystemExit(
+                "a real run requires --image-ref, --gpu-uuid, and --approved-inputs-dir; "
+                "use --fake only for a quarantined contract smoke"
+            )
+        executor = DockerTrainingExecutor(
+            image_ref=args.image_ref,
+            gpu_uuid=args.gpu_uuid,
+            approved_inputs_dir=args.approved_inputs_dir,
+            stream_output=args.stream_output,
+        )
+    runner = WakewordTrainingRunner(
+        settings=RunnerSettings(
+            work_root=args.work_root,
+            enabled=args.enable,
+            maintenance_window=StaticMaintenanceWindow(args.maintenance_window_active),
+            process_inspector=(
+                StaticGpuProcessInspector(active_unapproved_processes=False)
+                if args.fake or args.allow_active_gpu_processes
+                else NvidiaSmiProcessInspector(args.gpu_uuid)
+            ),
+            gpu_uuid=None if args.fake else args.gpu_uuid,
+            gpu_sampler=None if args.fake else NvidiaSmiGpuEvidenceSampler(args.gpu_uuid),
+            timeout_s=args.timeout_s,
+        ),
+        executor=executor,
+    )
+    result = await runner.run(request, retry_of=args.retry_of)
+    print(
+        json.dumps(
+            {
+                "run_dir": str(result.run_dir),
+                "run_id": result.state.run_id,
+                "status": result.state.status,
+            },
+            sort_keys=True,
+        )
+    )
+    return terminal_exit_code(result.state.status)
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(_main(_parser().parse_args())))