MC control-plane watchdog latest

Checked at: 2026-05-25T02:19:42.312171+00:00 Status: ISSUES

Issues

• WARNING mc4122_skip_loop: MC-4122 policy-skip loop still active
• Observed 82 pickup skips in the last ~2h. This is tracked by MC-4141.
• CRITICAL orchestrator_inbox_backlog: Orchestrator inbox pending backlog older than 30m
• pending=2, oldest_age_min=120.65520285
• WARNING green_but_skipped_review: Review task completed while skipping review
• [{"id": 286095, "task_id": "mc-auto-review", "status": "completed", "started_at": "2026-05-24T06:02:31.868304+02:00", "finished_at": "2026-05-24T06:02:32.842542+02:00", "output": "MC Auto-Review \u2014 2026-05-24 06:02 SAST\nChanged repos: workspace, mission-control \u2014 41969 diff lines\nMC Auto-Review skipped: diff is 41969 lines (cap 5000). Repos: workspace, mission-control. Run manually with --force to review anyway.\n", "age_min": 1337.2}]

Key facts

{
  "checked_at": "2026-05-25T02:19:42.312171+00:00",
  "service_luci-dashboard.service": "active",
  "service_ccgram.service": "active",
  "mc_api_probe": "HTTP 200",
  "task_success": {
    "luci-operator": {
      "run_id": 295199,
      "status": "completed",
      "finished_at": "2026-05-25T04:07:52.699468+02:00",
      "age_min": 11.8
    },
    "ticket-pickup": {
      "run_id": 295307,
      "status": "completed",
      "finished_at": "2026-05-25T04:19:03.670180+02:00",
      "age_min": 0.6
    },
    "needs-input-pickup": {
      "run_id": 295305,
      "status": "completed",
      "finished_at": "2026-05-25T04:19:01.391778+02:00",
      "age_min": 0.7
    },
    "triage-untriaged": {
      "run_id": 295308,
      "status": "completed",
      "finished_at": "2026-05-25T04:19:03.783467+02:00",
      "age_min": 0.6
    },
    "orchestrator-board-sweep": {
      "run_id": 294905,
      "status": "completed",
      "finished_at": "2026-05-25T03:20:01.401739+02:00",
      "age_min": 59.7
    },
    "persistent-luci-watchdog": {
      "run_id": 295277,
      "status": "completed",
      "finished_at": "2026-05-25T04:15:04.706507+02:00",
      "age_min": 4.6
    }
  },
  "recent_core_failures_2h": [],
  "mc_4122_skips_2h": 82,
  "stale_in_progress_gt_2h": [],
  "old_in_review_gt_4h": [],
  "orchestrator_inbox_pending": {
    "count": 2,
    "oldest_age_min": 120.65520285
  },
  "green_but_skipped_reviews_24h": [
    {
      "id": 286095,
      "task_id": "mc-auto-review",
      "status": "completed",
      "started_at": "2026-05-24T06:02:31.868304+02:00",
      "finished_at": "2026-05-24T06:02:32.842542+02:00",
      "output": "MC Auto-Review \u2014 2026-05-24 06:02 SAST\nChanged repos: workspace, mission-control \u2014 41969 diff lines\nMC Auto-Review skipped: diff is 41969 lines (cap 5000). Repos: workspace, mission-control. Run manually with --force to review anyway.\n",
      "age_min": 1337.2
    }
  ],
  "mc_4141": {
    "identifier": "MC-4141",
    "status": "in_progress",
    "assigned_to": "luci",
    "updated_at": "2026-05-25T03:02:25.280084+02:00",
    "title": "Fix MC operator/pickup reliability based on outside review"
  }
}

Tracked repair ticket: http://100.118.207.3:3001/ticket/4141