113 lines
4.0 KiB
Python
Executable File
113 lines
4.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Start one sealed offline Wakeword Training attempt.
|
|
|
|
The command is deliberately operator-only. It has no gateway/API-key path and
|
|
refuses to start unless both ``--enable`` and an active maintenance window are
|
|
explicitly supplied. See docs/wakeword-training-operator-runner.md.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from wakeword_training import (
|
|
DockerTrainingExecutor,
|
|
FakeTrainingExecutor,
|
|
NvidiaSmiGpuEvidenceSampler,
|
|
NvidiaSmiProcessInspector,
|
|
RunnerSettings,
|
|
StaticGpuProcessInspector,
|
|
StaticMaintenanceWindow,
|
|
WakewordTrainingRequest,
|
|
WakewordTrainingRunner,
|
|
terminal_exit_code,
|
|
)
|
|
|
|
|
|
def _parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--request", required=True, type=Path, help="immutable request JSON")
|
|
parser.add_argument("--work-root", required=True, type=Path, help="operator run root")
|
|
parser.add_argument("--enable", action="store_true", help="explicitly permit this invocation")
|
|
parser.add_argument(
|
|
"--maintenance-window-active",
|
|
action="store_true",
|
|
help="attest that the configured GPU maintenance window is active",
|
|
)
|
|
parser.add_argument("--retry-of", help="terminal matching run ID for a new immutable attempt")
|
|
parser.add_argument("--timeout-s", type=float, default=6 * 60 * 60)
|
|
parser.add_argument(
|
|
"--fake", action="store_true", help="CI/local quarantined contract executor"
|
|
)
|
|
parser.add_argument(
|
|
"--image-ref", help="digest-pinned training image, e.g. registry/image@sha256:…"
|
|
)
|
|
parser.add_argument("--gpu-uuid", help="stable configured RTX UUID, never a mutable ordinal")
|
|
parser.add_argument(
|
|
"--approved-inputs-dir",
|
|
type=Path,
|
|
help="hash-locked data mount; mounted read-only at /inputs",
|
|
)
|
|
parser.add_argument(
|
|
"--allow-active-gpu-processes",
|
|
action="store_true",
|
|
help="explicit shared-GPU override; never terminates another process",
|
|
)
|
|
parser.add_argument(
|
|
"--stream-output", action="store_true", help="mirror container progress to stderr"
|
|
)
|
|
return parser
|
|
|
|
|
|
async def _main(args: argparse.Namespace) -> int:
|
|
request = WakewordTrainingRequest.from_file(args.request)
|
|
if args.fake:
|
|
executor = FakeTrainingExecutor()
|
|
else:
|
|
if not (args.image_ref and args.gpu_uuid and args.approved_inputs_dir):
|
|
raise SystemExit(
|
|
"a real run requires --image-ref, --gpu-uuid, and --approved-inputs-dir; "
|
|
"use --fake only for a quarantined contract smoke"
|
|
)
|
|
executor = DockerTrainingExecutor(
|
|
image_ref=args.image_ref,
|
|
gpu_uuid=args.gpu_uuid,
|
|
approved_inputs_dir=args.approved_inputs_dir,
|
|
stream_output=args.stream_output,
|
|
)
|
|
runner = WakewordTrainingRunner(
|
|
settings=RunnerSettings(
|
|
work_root=args.work_root,
|
|
enabled=args.enable,
|
|
maintenance_window=StaticMaintenanceWindow(args.maintenance_window_active),
|
|
process_inspector=(
|
|
StaticGpuProcessInspector(active_unapproved_processes=False)
|
|
if args.fake or args.allow_active_gpu_processes
|
|
else NvidiaSmiProcessInspector(args.gpu_uuid)
|
|
),
|
|
gpu_uuid=None if args.fake else args.gpu_uuid,
|
|
gpu_sampler=None if args.fake else NvidiaSmiGpuEvidenceSampler(args.gpu_uuid),
|
|
timeout_s=args.timeout_s,
|
|
),
|
|
executor=executor,
|
|
)
|
|
result = await runner.run(request, retry_of=args.retry_of)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"run_dir": str(result.run_dir),
|
|
"run_id": result.state.run_id,
|
|
"status": result.state.status,
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
return terminal_exit_code(result.state.status)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(asyncio.run(_main(_parser().parse_args())))
|