feat: collect minio logs

[nvanthao]

Story: https://app.shortcut.com/replicated/story/116844/in-cluster-default-troubleshoot-spec-does-not-collect-kotsadm-minio-logs

• Collect logs from kotsadm minio pod
• Collect summary of objects in minio

E.g.

{"status":"success","type":"file","lastModified":"2024-12-09T00:39:22.543254192Z","size":256589,"key":"2pxONEd1RFZgERXgDun51yAFAvV/0.tar.gz/3c7ec8a6-c8d5-4664-b09e-f21f35ae780a/part.1","etag":"","url":"/export/kotsadm/","versionOrdinal":1}
{"status":"success","type":"file","lastModified":"2024-12-09T00:39:22.557254301Z","size":360,"key":"2pxONEd1RFZgERXgDun51yAFAvV/0.tar.gz/xl.meta","etag":"","url":"/export/kotsadm/","versionOrdinal":1}
{"status":"success","type":"file","lastModified":"2024-12-09T00:39:52.425492415Z","size":256589,"key":"2pxONEd1RFZgERXgDun51yAFAvV/1.tar.gz/33aa5b39-1801-41b8-a746-c695aeb44778/part.1","etag":"","url":"/export/kotsadm/","versionOrdinal":1}
{"status":"success","type":"file","lastModified":"2024-12-09T00:39:52.431492499Z","size":360,"key":"2pxONEd1RFZgERXgDun51yAFAvV/1.tar.gz/xl.meta","etag":"","url":"/export/kotsadm/","versionOrdinal":1}
{"totalObjects":4,"totalSize":513898}

[hedge-sparrow]

this exec collector seems useful, could we get this added for the non kotsadm-minio as well?

[nvanthao]

thanks @hedge-sparrow, do you mean the minio-fs for hostpath snapshot? I've updated the spec accordingly.

[nvanthao]

also added mc admin info, which will give us all we need:

• is MinIO up?
• bucket size

{
  "status": "success",
  "info": {
    "mode": "online",
    "deploymentID": "b77058ed-8280-4155-871b-d6dab17e70fe",
    "buckets": {
      "count": 1
    },
    "objects": {
      "count": 2
    },
    "versions": {
      "count": 0
    },
    "deletemarkers": {
      "count": 0
    },
    "usage": {
      "size": 138310
    },
    "services": {
      "kms": {
        "status": "disabled"
      },
      "ldap": {}
    },
    "backend": {
      "backendType": "Erasure",
      "onlineDisks": 1,
      "offlineDisks": 0,
      "standardSCParity": 0,
      "rrSCParity": 0,
      "totalSets": [
        1
      ],
      "totalDrivesPerSet": [
        1
      ]
    },
    "servers": [
      {
        "state": "online",
        "endpoint": "localhost:9000",
        "uptime": 16970,
        "version": "2023-12-20T01:00:02Z",
        "commitID": "7a311a3b666a0e5476fbde24d4413220876d9df2",
        "network": {
          "localhost:9000": "online"
        },
        "drives": [
          {
            "endpoint": "/data",
            "path": "/data",
            "state": "ok",
            "uuid": "825eef3e-c01c-49b3-aac8-ead7a8cff7d0",
            "major": 254,
            "minor": 1,
            "totalspace": 59454382080,
            "usedspace": 17296285696,
            "availspace": 42158096384,
            "metrics": {
              "lastMinute": {
                "Delete": {
                  "count": 6,
                  "acc_time_ns": 962084
                },
                "DiskInfo": {
                  "count": 2,
                  "acc_time_ns": 2949542
                },
                "ListVols": {
                  "count": 1,
                  "acc_time_ns": 281416
                },
                "ReadXL": {
                  "count": 2,
                  "acc_time_ns": 102125
                },
                "RenameData": {
                  "count": 6,
                  "acc_time_ns": 14420252
                },
                "StatVol": {
                  "count": 1,
                  "acc_time_ns": 115584
                }
              },
              "apiCalls": {
                "AppendFile": 0,
                "CheckParts": 0,
                "CreateFile": 0,
                "Delete": 1262,
                "DeleteAbandonedParts": 0,
                "DeleteVersion": 0,
                "DeleteVersions": 0,
                "DeleteVol": 0,
                "DiskInfo": 788,
                "ListDir": 0,
                "ListVols": 215,
                "MakeVol": 0,
                "MakeVolBulk": 0,
                "ReadAll": 2,
                "ReadFile": 0,
                "ReadFileStream": 0,
                "ReadMultiple": 0,
                "ReadVersion": 2,
                "ReadXL": 680,
                "RenameData": 1262,
                "RenameFile": 0,
                "StatInfoFile": 0,
                "StatVol": 164,
                "UpdateMetadata": 0,
                "VerifyFile": 0,
                "WalkDir": 141,
                "WriteAll": 0,
                "WriteMetadata": 0
              }
            },
            "used_inodes": 63077,
            "free_inodes": 3844507,
            "pool_index": 0,
            "set_index": 0,
            "disk_index": 0
          }
        ],
        "mem_stats": {
          "Alloc": 260644840,
          "TotalAlloc": 1488059600,
          "Mallocs": 1822608,
          "Frees": 1745546,
          "HeapAlloc": 260644840
        },
        "go_max_procs": 11,
        "num_cpu": 11,
        "runtime_version": "go1.21.6",
        "gc_stats": {
          "last_gc": "2024-12-10T05:08:32.492807387Z",
          "num_gc": 111,
          "pause_total": 205577461,
          "pause": [
            1321333,
            39916,
            160124,
            1004625,
            624833
          ],
          "pause_end": [
            "2024-12-10T00:25:45.833274508Z",
            "2024-12-10T00:25:45.824463716Z",
            "2024-12-10T00:25:45.796226258Z",
            "2024-12-10T00:25:45.759222341Z",
            "2024-12-10T00:25:45.631859466Z"
          ]
        },
        "minio_env_vars": {
          "MINIO_ACCESS_KEY": "*** EXISTS, REDACTED ***",
          "MINIO_BROWSER": "on",
          "MINIO_SECRET_KEY": "*** EXISTS, REDACTED ***"
        }
      }
    ],
    "pools": {
      "0": {
        "0": {
          "id": 0,
          "rawUsage": 17296285696,
          "rawCapacity": 59454382080,
          "usage": 138310,
          "objectsCount": 2,
          "versionsCount": 0,
          "deleteMarkersCount": 0,
          "healDisks": 0
        }
      }
    }
  }
}