#!/usr/bin/env python3 """Start one sealed offline Wakeword Training attempt. The command is deliberately operator-only. It has no gateway/API-key path and refuses to start unless both ``--enable`` and an active maintenance window are explicitly supplied. See docs/wakeword-training-operator-runner.md. """ from __future__ import annotations import argparse import asyncio import json from pathlib import Path from wakeword_training import ( DockerTrainingExecutor, FakeTrainingExecutor, NvidiaSmiGpuEvidenceSampler, NvidiaSmiProcessInspector, RunnerSettings, StaticGpuProcessInspector, StaticMaintenanceWindow, WakewordTrainingRequest, WakewordTrainingRunner, terminal_exit_code, ) def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--request", required=True, type=Path, help="immutable request JSON") parser.add_argument("--work-root", required=True, type=Path, help="operator run root") parser.add_argument("--enable", action="store_true", help="explicitly permit this invocation") parser.add_argument( "--maintenance-window-active", action="store_true", help="attest that the configured GPU maintenance window is active", ) parser.add_argument("--retry-of", help="terminal matching run ID for a new immutable attempt") parser.add_argument("--timeout-s", type=float, default=6 * 60 * 60) parser.add_argument( "--fake", action="store_true", help="CI/local quarantined contract executor" ) parser.add_argument( "--image-ref", help="digest-pinned training image, e.g. registry/image@sha256:…" ) parser.add_argument("--gpu-uuid", help="stable configured RTX UUID, never a mutable ordinal") parser.add_argument( "--approved-inputs-dir", type=Path, help="hash-locked data mount; mounted read-only at /inputs", ) parser.add_argument( "--allow-active-gpu-processes", action="store_true", help="explicit shared-GPU override; never terminates another process", ) parser.add_argument( "--stream-output", action="store_true", help="mirror container progress to stderr" ) return parser async def _main(args: argparse.Namespace) -> int: request = WakewordTrainingRequest.from_file(args.request) if args.fake: executor = FakeTrainingExecutor() else: if not (args.image_ref and args.gpu_uuid and args.approved_inputs_dir): raise SystemExit( "a real run requires --image-ref, --gpu-uuid, and --approved-inputs-dir; " "use --fake only for a quarantined contract smoke" ) executor = DockerTrainingExecutor( image_ref=args.image_ref, gpu_uuid=args.gpu_uuid, approved_inputs_dir=args.approved_inputs_dir, stream_output=args.stream_output, ) runner = WakewordTrainingRunner( settings=RunnerSettings( work_root=args.work_root, enabled=args.enable, maintenance_window=StaticMaintenanceWindow(args.maintenance_window_active), process_inspector=( StaticGpuProcessInspector(active_unapproved_processes=False) if args.fake or args.allow_active_gpu_processes else NvidiaSmiProcessInspector(args.gpu_uuid) ), gpu_uuid=None if args.fake else args.gpu_uuid, gpu_sampler=None if args.fake else NvidiaSmiGpuEvidenceSampler(args.gpu_uuid), timeout_s=args.timeout_s, ), executor=executor, ) result = await runner.run(request, retry_of=args.retry_of) print( json.dumps( { "run_dir": str(result.run_dir), "run_id": result.state.run_id, "status": result.state.status, }, sort_keys=True, ) ) return terminal_exit_code(result.state.status) if __name__ == "__main__": raise SystemExit(asyncio.run(_main(_parser().parse_args())))