105 lines
2.6 KiB
Bash
Executable file
105 lines
2.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# Trigger knowledge model training on VPS via systemd
|
|
#
|
|
# This script is called by:
|
|
# 1. Forgejo Actions (via SSH)
|
|
# 2. Cron job (scheduled check)
|
|
# 3. Manual invocation
|
|
#
|
|
# Usage:
|
|
# ./trigger-training-vps.sh # Check cooldown first
|
|
# ./trigger-training-vps.sh --force # Bypass cooldown
|
|
# ./trigger-training-vps.sh --status # Check training status
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SERVICE_NAME="crystal-train.service"
|
|
TRAINING_MARKER="/var/home/lilith/.cache/crystal/last-training-run"
|
|
LOG_FILE="/var/home/lilith/.cache/crystal/training.log"
|
|
|
|
# Parse arguments
|
|
FORCE=false
|
|
CHECK_STATUS=false
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--force)
|
|
FORCE=true
|
|
shift
|
|
;;
|
|
--status)
|
|
CHECK_STATUS=true
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1"
|
|
echo "Usage: $0 [--force] [--status]"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Check status only
|
|
if [[ "$CHECK_STATUS" == "true" ]]; then
|
|
echo "=== Training Status ==="
|
|
echo ""
|
|
|
|
# Check if service is running
|
|
if systemctl is-active --quiet "$SERVICE_NAME"; then
|
|
echo "Status: RUNNING"
|
|
echo ""
|
|
echo "Recent logs:"
|
|
journalctl -u "$SERVICE_NAME" -n 20 --no-pager
|
|
else
|
|
echo "Status: IDLE"
|
|
|
|
if [[ -f "$TRAINING_MARKER" ]]; then
|
|
last_trained=$(stat -c %y "$TRAINING_MARKER" 2>/dev/null || stat -f %Sm "$TRAINING_MARKER")
|
|
echo "Last trained: $last_trained"
|
|
else
|
|
echo "Last trained: never"
|
|
fi
|
|
fi
|
|
|
|
exit 0
|
|
fi
|
|
|
|
# Check cooldown unless forced
|
|
if [[ "$FORCE" == "false" ]]; then
|
|
FORCE_TRAINING=false COOLDOWN_HOURS=6 bash "$(dirname "$0")/check-training-needed.sh" > /tmp/training-check.txt
|
|
|
|
if grep -q "should_train=false" /tmp/training-check.txt; then
|
|
echo "Training skipped - cooldown active"
|
|
echo "Use --force to bypass cooldown"
|
|
cat /tmp/training-check.txt
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# Check if already running
|
|
if systemctl is-active --quiet "$SERVICE_NAME"; then
|
|
echo "ERROR: Training is already running"
|
|
echo "Check status with: systemctl status $SERVICE_NAME"
|
|
exit 1
|
|
fi
|
|
|
|
# Trigger training
|
|
echo "=== Triggering Knowledge Model Training ==="
|
|
echo ""
|
|
echo "Service: $SERVICE_NAME"
|
|
echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "Force: $FORCE"
|
|
echo ""
|
|
|
|
# Start the systemd service
|
|
sudo systemctl start "$SERVICE_NAME"
|
|
|
|
echo "Training started successfully!"
|
|
echo ""
|
|
echo "Monitor progress with:"
|
|
echo " journalctl -u $SERVICE_NAME -f"
|
|
echo ""
|
|
echo "Check status with:"
|
|
echo " $0 --status"
|