Merge remote-tracking branch 'upstream/main' into chtruong/remove-transition-msg

Signed-off-by: Charlie Truong <chtruong@nvidia.com>

Author: chtruong814

.github/actions/action.yml

@@ -11,28 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-name: 'Test Template'
-description: 'Template for running NeMo tests in a containerized environment'
+name: "Test Template"
+description: "Template for running NeMo tests in a containerized environment"
 
 inputs:
   container-image:
-    description: 'Container image to use for test'
+    description: "Container image to use for test"
     required: true
   timeout:
-    description: 'Max runtime of test in minutes'
+    description: "Max runtime of test in minutes"
     required: false
-    default: '30'
+    default: "30"
   script:
-    description: 'Test script to execute'
+    description: "Test script to execute"
     required: true
   is-optional:
-    description: 'Pass this job on failure.'
+    description: "Pass this job on failure."
     required: false
-    default: 'false'
+    default: "false"
   is_unit_test:
-    description: 'Upload coverage as unit test'
+    description: "Upload coverage as unit test"
     required: false
-    default: 'false'
+    default: "false"
   tag:
     description: Latest or legacy test suite
     required: true
@@ -43,11 +43,14 @@ inputs:
     description: Model to launch
     required: false
   PAT:
-    description: 'GitHub Personal Access Token'
+    description: "GitHub Personal Access Token"
+    required: true
+  is_ci_workload:
+    description: "Is CI workload"
     required: true
 
 runs:
-  using: 'composite'
+  using: "composite"
   steps:
     - name: Checkout repository
       uses: actions/checkout@v2
@@ -77,6 +80,7 @@ runs:
 
         export PYTHONPATH=$(pwd)
         export NEMORUN_HOME=$(pwd)
+        export NCCL_DEBUG=INFO
         pip install --no-cache-dir uv
         uv sync --only-group test
         uv run python tests/test_utils/python_scripts/launch_nemo_run_workload.py \
@@ -119,9 +123,11 @@ runs:
       id: has-run-functional-tests-label
       env:
         GH_TOKEN: ${{ github.token }}
+        IS_CI_WORKLOAD: ${{ inputs.is_ci_workload }}
       run: |
         PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
-        HAS_RUN_FUNCTIONAL_TESTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "Run functional tests")') || echo "false"
+        HAS_RUN_FUNCTIONAL_TESTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "Run functional tests")') || echo "$IS_CI_WORKLOAD"
+        HAS_RUN_FUNCTIONAL_TESTS_LABEL=${HAS_RUN_FUNCTIONAL_TESTS_LABEL:-$IS_CI_WORKLOAD}
         echo "main=$HAS_RUN_FUNCTIONAL_TESTS_LABEL" | tee -a $GITHUB_OUTPUT
 
     - name: Create run-script (e2e test)

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
 enabled: true
 auto_sync_draft: false
 auto_sync_ready: true
-trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Phlip79", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "frsun-nvda", "gautham-kollu", "gdengk", "guyueh1", "hxbai", "jalbericiola", "jaredcasper", "jenchen13", "jiemingz", "jkamalu", "jon-barker", "jstjohn", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mehraakash", "mkhona-nvidia", "pablo-garay", "parthmannan", "pthombre", "rogerwaleffe", "sanandaraj5597", "santhnm2", "sbak5", "shanmugamr1992", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"]
+trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Phlip79", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "frsun-nvda", "gautham-kollu", "gdengk", "guyueh1", "hxbai", "jalbericiola", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mehraakash", "mkhona-nvidia", "pablo-garay", "parthmannan", "pthombre", "rogerwaleffe", "sanandaraj5597", "santhnm2", "sbak5", "shanmugamr1992", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"]

.github/oncall_schedule.json

@@ -1,30 +1,50 @@
 [
     {
-        "user": "Phlip79",
-        "date": "2026-01-07"
+        "user": "maanug-nv",
+        "date": "2026-01-21"
     },
     {
-        "user": "BoxiangW",
-        "date": "2026-01-14"
+        "user": "dimapihtar",
+        "date": "2026-01-28"
     },
     {
-        "user": "maanug-nv",
-        "date": "2026-01-21"
+        "user": "gautham-kollu",
+        "date": "2026-02-04"
+    },
+    {
+        "user": "janEbert",
+        "date": "2026-02-11"
+    },
+    {
+        "user": "Phlip79",
+        "date": "2026-02-18"
     },
     {
         "user": "asolergi-nv",
-        "date": "2026-01-28"
+        "date": "2026-02-25"
+    },
+    {
+        "user": "BoxiangW",
+        "date": "2026-03-04"
+    },
+    {
+        "user": "maanug-nv",
+        "date": "2026-03-11"
     },
     {
         "user": "dimapihtar",
-        "date": "2026-02-04"
+        "date": "2026-03-18"
     },
     {
         "user": "gautham-kollu",
-        "date": "2026-02-11"
+        "date": "2026-03-25"
     },
     {
         "user": "janEbert",
-        "date": "2026-02-18"
+        "date": "2026-04-01"
+    },
+    {
+        "user": "maanug-nv",
+        "date": "2026-04-08"
     }
 ]

.github/scripts/oncall_manager.py

@@ -19,13 +19,21 @@
 import argparse
 from datetime import datetime, timedelta, timezone
 
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+
 # Constants
 GITHUB_API_URL = "https://api.github.com"
 SCHEDULE_FILE = ".github/oncall_schedule.json"
 ROTATION_TEAM_SLUG = "mcore-oncall-rotation"
 ACTIVE_ONCALL_TEAM_SLUG = "mcore-oncall"
+SLACK_USERGROUP_HANDLE = "mcore-oncall"
 TARGET_WEEKS = 12
 
+# Caches for email and Slack lookups
+_email_cache = {}
+_slack_id_cache = {}
+
 def get_headers():
     token = os.environ.get("GH_TOKEN")
     if not token:
@@ -74,6 +82,159 @@ def get_team_members(org, team_slug):
 
     return members
 
+def get_user_email(username):
+    """Get user's email from GitHub, prioritizing @nvidia.com emails.
+    
+    Checks in order:
+    1. Public profile email
+    2. Recent commits in the repository
+    """
+    if username in _email_cache:
+        return _email_cache[username]
+    
+    headers = get_headers()
+    public_email = None
+    
+    try:
+        # 1. Try to get user's public profile email first
+        resp = requests.get(f"{GITHUB_API_URL}/users/{username}", headers=headers)
+        if resp.status_code == 200:
+            user_data = resp.json()
+            email = user_data.get('email')
+            if email and not email.endswith("@users.noreply.github.com"):
+                if email.endswith("@nvidia.com"):
+                    _email_cache[username] = email
+                    return email
+                # Store non-nvidia email as fallback
+                public_email = email
+        
+        # 2. Check recent commits in the repository for @nvidia.com email
+        repo_env = os.environ.get("GITHUB_REPOSITORY", "NVIDIA/Megatron-LM")
+        commits_url = f"{GITHUB_API_URL}/repos/{repo_env}/commits?author={username}&per_page=10"
+        resp = requests.get(commits_url, headers=headers)
+        
+        if resp.status_code == 200:
+            commits = resp.json()
+            for commit in commits:
+                # Get email from commit author
+                commit_data = commit.get('commit', {})
+                author_data = commit_data.get('author', {})
+                email = author_data.get('email')
+                
+                if email and not email.endswith("@users.noreply.github.com"):
+                    if email.endswith("@nvidia.com"):
+                        _email_cache[username] = email
+                        print(f"Found @nvidia.com email for {username} from commits: {email}")
+                        return email
+                    elif public_email is None:
+                        public_email = email
+        
+        # 3. Use public email if found, otherwise fallback
+        if public_email:
+            _email_cache[username] = public_email
+            print(f"Using public email for {username}: {public_email}")
+            return public_email
+        
+        # Fallback to noreply email
+        fallback = f"{username}@users.noreply.github.com"
+        _email_cache[username] = fallback
+        print(f"Warning: No email found for {username}, using fallback: {fallback}")
+        return fallback
+        
+    except Exception as e:
+        print(f"Warning: Could not get email for {username}: {e}")
+        fallback = f"{username}@users.noreply.github.com"
+        _email_cache[username] = fallback
+        return fallback
+
+def get_slack_client():
+    """Get Slack WebClient if token is available."""
+    slack_token = os.environ.get("SLACK_TOKEN")
+    if not slack_token:
+        return None
+    
+    return WebClient(token=slack_token)
+
+def get_slack_user_id(slack_client, email):
+    """Get Slack user ID from email."""
+    if not slack_client:
+        return None
+    
+    if email in _slack_id_cache:
+        return _slack_id_cache[email]
+    
+    try:
+        response = slack_client.users_lookupByEmail(email=email)
+        user_id = response["user"]["id"]
+        _slack_id_cache[email] = user_id
+        return user_id
+    except SlackApiError as e:
+        print(f"Warning: Could not find Slack user for {email}: {e.response['error']}")
+        _slack_id_cache[email] = None
+        return None
+
+def get_slack_usergroup_id(slack_client, handle):
+    """Get Slack usergroup ID from handle."""
+    if not slack_client:
+        return None
+    
+    try:
+        response = slack_client.usergroups_list(include_users=True)
+        for usergroup in response.get("usergroups", []):
+            if usergroup.get("handle") == handle:
+                return usergroup.get("id"), usergroup.get("users", [])
+        print(f"Warning: Slack usergroup '{handle}' not found")
+        return None, []
+    except SlackApiError as e:
+        print(f"Warning: Could not list Slack usergroups: {e.response['error']}")
+        return None, []
+
+def update_slack_usergroup(new_oncall_username, old_members_usernames):
+    """
+    Updates the Slack usergroup to contain only the new oncall user.
+    Adds new oncall first, then removes old members (usergroups need at least one member).
+    """
+    slack_client = get_slack_client()
+    if not slack_client:
+        print("Slack token not configured, skipping Slack usergroup update")
+        return
+    
+    # Get the new oncall's email and Slack user ID
+    new_email = get_user_email(new_oncall_username)
+    new_slack_id = get_slack_user_id(slack_client, new_email)
+    
+    if not new_slack_id:
+        print(f"Could not find Slack user ID for {new_oncall_username} ({new_email}), skipping Slack update")
+        return
+    
+    # Get the usergroup ID and current members
+    usergroup_id, current_slack_members = get_slack_usergroup_id(slack_client, SLACK_USERGROUP_HANDLE)
+    
+    if not usergroup_id:
+        print(f"Could not find Slack usergroup '{SLACK_USERGROUP_HANDLE}', skipping Slack update")
+        return
+    
+    try:
+        # Step 1: Add new oncall first (include current members to avoid removing anyone yet)
+        # This ensures usergroup always has at least one member
+        if new_slack_id not in current_slack_members:
+            updated_members = list(set(current_slack_members + [new_slack_id]))
+            slack_client.usergroups_users_update(
+                usergroup=usergroup_id,
+                users=updated_members
+            )
+            print(f"Added {new_oncall_username} to Slack usergroup '{SLACK_USERGROUP_HANDLE}'")
+        
+        # Step 2: Now set the usergroup to contain only the new oncall
+        slack_client.usergroups_users_update(
+            usergroup=usergroup_id,
+            users=[new_slack_id]
+        )
+        print(f"Updated Slack usergroup '{SLACK_USERGROUP_HANDLE}' to contain only {new_oncall_username}")
+        
+    except SlackApiError as e:
+        print(f"Failed to update Slack usergroup: {e.response['error']}")
+
 def load_schedule():
     if not os.path.exists(SCHEDULE_FILE):
         return []
@@ -111,14 +272,19 @@ def update_active_oncall_team(org, new_oncall):
             print(f"Failed to add {new_oncall} to {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}")
 
     # 3. Remove everyone else
+    old_members = []
     for member in current_members:
         if member not in [new_oncall, 'svcnvidia-nemo-ci']:
+            old_members.append(member)
             url = f"{GITHUB_API_URL}/orgs/{org}/teams/{ACTIVE_ONCALL_TEAM_SLUG}/memberships/{member}"
             resp = requests.delete(url, headers=get_headers())
             if resp.status_code == 204:
                 print(f"Removed {member} from {ACTIVE_ONCALL_TEAM_SLUG}")
             else:
                 print(f"Failed to remove {member} from {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}")
+    
+    # 4. Update Slack usergroup (add new oncall first, then remove old members)
+    update_slack_usergroup(new_oncall, old_members)
 
 def rotate_schedule(repo_owner, dry_run=False):
     schedule = load_schedule()
@@ -225,25 +391,16 @@ def ensure_schedule_filled(schedule, repo_owner):
         print(f"Appended: {new_entry}")
 
 def assign_reviewer(pr_number):
-    """Assigns the current oncall as the reviewer for the PR."""
-    schedule = load_schedule()
-    if not schedule:
-        print("Error: Schedule is empty. Cannot assign reviewer.")
-        sys.exit(1)
-        
-    current_entry = schedule[0]
-    current_oncall = current_entry['user']
-    print(f"Current oncall: {current_oncall} (Since {current_entry['date']})")
-    
+    """Assigns the mcore-oncall team as the reviewer for the PR."""
     owner, repo = get_repo_info()
     url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{pr_number}/requested_reviewers"
 
-    # We can assign the user directly
-    data = {"reviewers": [current_oncall]}
+    # Assign the oncall team as reviewer
+    data = {"team_reviewers": [ACTIVE_ONCALL_TEAM_SLUG]}
     resp = requests.post(url, headers=get_headers(), json=data)
 
     if resp.status_code in [201, 200]:
-        print(f"Successfully requested review from {current_oncall}")
+        print(f"Successfully requested review from team NVIDIA/{ACTIVE_ONCALL_TEAM_SLUG}")
     else:
         print(f"Failed to request review: {resp.status_code} {resp.text}")
         sys.exit(1)