#!/bin/bash

# WLP Agent Missing Heartbeat Diagnostic Script for Linux
# Usage: sudo ./wlp-doctor
# 
# This script detects environment and OS image type to run appropriate diagnostics:
#
# OS IMAGE TYPES:
#   - NON-EVERGREEN: Standard Linux with mutable package management
#   - EVERGREEN: Hardened rpm-ostree based immutable OS image
#
# POSSIBLE COMBINATIONS:
#   - Non-Evergreen Overlay: Standard OCI instance
#   - Non-Evergreen Substrate: Chef-managed standard instance (uses CVA-Updater)
#   - Evergreen Overlay: Evergreen OS on standard OCI (container-based WLP)
#   - Evergreen Substrate: Evergreen OS on Chef-managed infrastructure (container-based WLP)
#
# IMPORTANT NOTES:
#   - CVA-Updater is ONLY used on Non-Evergreen Substrate systems
#   - Evergreen systems (both Overlay and Substrate) use containers, NOT CVA-Updater
#
# Detection logic:
#   1. Environment: Check /etc/sccp-instance.json → Substrate vs Overlay
#   2. OS Image: Check /etc/evergreen-release.json or rpm-ostree → Evergreen vs Non-Evergreen
#   3. Run diagnostics based on both environment and OS image type
#
# Ubuntu checks (20/22/23/24) apply to all environment combinations

# Check if running on Linux 
if [[ "$(uname -s)" != "Linux" ]]; then
    echo "[ERROR] This script is designed for Linux systems only"
    echo "Current OS: $(uname -s)"
    echo "Please run this script on a Linux system."
    exit 1
fi

# Setup simple logging and collection
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")

# Function to find a writable directory for diagnosis output
find_writable_dir() {
    local dir_name="wlp_diagnosis_${TIMESTAMP}"
    local candidate_dirs=("/tmp" "/var/tmp" "$HOME" ".")
    
    for base_dir in "${candidate_dirs[@]}"; do
        # Skip if base directory doesn't exist
        [[ ! -d "$base_dir" ]] && continue
        
        # Try to create the diagnosis directory
        local test_dir="${base_dir}/${dir_name}"
        if mkdir -p "$test_dir" 2>/dev/null; then
            # Verify we can write to it
            if touch "${test_dir}/.write_test" 2>/dev/null; then
                rm -f "${test_dir}/.write_test"
                echo "$test_dir"
                return 0
            else
                rmdir "$test_dir" 2>/dev/null
            fi
        fi
    done
    
    return 1
}

# Find a writable directory for output
OUTPUT_DIR=$(find_writable_dir)

if [[ -z "$OUTPUT_DIR" ]]; then
    echo "[ERROR] Cannot find a writable directory for diagnosis output."
    echo "[ERROR] Tried: /tmp, /var/tmp, $HOME, current directory"
    echo "[ACTION] Please ensure at least one of these locations is writable, or run from a writable directory."
    exit 1
fi

# Start logging all output to file
exec > >(tee "${OUTPUT_DIR}/diagnosis_output.log") 2>&1

echo "WLP Diagnosis started: $(date)"
echo "Output will be saved to: ${OUTPUT_DIR}/"

# Function to print section headers
print_header() {
    echo ""
    echo "---------------------------------------"
    echo "$1"
    echo "---------------------------------------"
}

print_double_header() {
    echo ""
    echo "=============================================="
    echo "$1"
    echo "=============================================="
}
# Function to show CVA-Updater troubleshooting info
# Note: CVA-Updater is ONLY used on Non-Evergreen Substrate systems
show_cva_updater_guidance() {
    echo ""
    echo "[INFO] CVA-Updater should be installed automatically by Chef on chef-managed instances"
    echo "[ISSUE] If CVA-Updater is not installed, there is likely a Chef configuration problem"
    echo ""
    echo "[ACTION] Run Chef doctor script to diagnose Chef failures:"
    echo "  https://devops.oci.oraclecorp.com/runbooks/CHEF/chef-how-tos/chef-diagnosing-chef-failures/chef-how-to-use-chef-doctor"
    echo ""
    echo "[INFO] CVA-Updater runs every hour"
    echo "[ACTION] If CVA-Updater is installed but has issues:"
    echo "  - Force run: sudo /opt/cva-updater/bin/cva-updater"
    echo "  - Check logs: /var/log/cva/cva-updater.log"
}


# Function to analyze WLP Linux log for all error types at once
analyze_wlp_linux_logs() {
    local log_file="$1"
    
    # Declare associative arrays to store different error types
    declare -A error_patterns=(
        ["io_timeout"]="giving up after 2 attempt\(s\): dial tcp .*: i/o timeout"
        ["dependency"]="\\[ERROR\\].*step.*wlpagent\\.InstallDependency.*execution failed"
        ["rpm_lock"]="execution failed with.*error.*can't create transaction lock on /var/lib/rpm/\\.rpm\\.lock.*Resource temporarily unavailable"
        ["pam_error"]="PAM account management error.*Permission denied"
        ["rpmdb_corruption"]="cannot open Packages|rpmdb.*error|Thread died in Berkeley DB|DB_RUNRECOVERY"
    )
    
    # Single grep to capture all errors at once
    local all_errors
    if [[ -f "$log_file" ]] && [[ -r "$log_file" ]]; then
        # Combine all patterns with OR operator and search once
        local combined_pattern=""
        for pattern in "${error_patterns[@]}"; do
            if [[ -n "$combined_pattern" ]]; then
                combined_pattern="${combined_pattern}|${pattern}"
            else
                combined_pattern="$pattern"
            fi
        done
        
        all_errors=$(sudo grep -E "$combined_pattern" "$log_file" 2>/dev/null || echo "")
    else
        all_errors=""
    fi
    
    # Now categorize the found errors
    for error_type in "${!error_patterns[@]}"; do
        if [[ -n "$all_errors" ]]; then
            local specific_errors=$(echo "$all_errors" | grep -E "${error_patterns[$error_type]}" || echo "")
            
            # Export the results as global variables for backward compatibility
            case "$error_type" in
                "io_timeout") export wlp_io_timeout_errors="$specific_errors" ;;
                "dependency") export wlp_dependency_errors="$specific_errors" ;;
                "rpm_lock") export wlp_rpm_lock_errors="$specific_errors" ;;
                "pam_error") export wlp_pam_errors="$specific_errors" ;;
                "rpmdb_corruption") export wlp_rpmdb_errors="$specific_errors" ;;
            esac
        else
            # No errors found, set empty values
            case "$error_type" in
                "io_timeout") export wlp_io_timeout_errors="" ;;
                "dependency") export wlp_dependency_errors="" ;;
                "rpm_lock") export wlp_rpm_lock_errors="" ;;
                "pam_error") export wlp_pam_errors="" ;;
                "rpmdb_corruption") export wlp_rpmdb_errors="" ;;
            esac
        fi
    done
}

# Function to compare version numbers using sort -V (fast and reliable)
# Returns 0 if version1 >= version2, 1 otherwise
# Usage: compare_versions "1.1.272" "1.1.220"
compare_versions() {
    local version1="$1"
    local version2="$2"
    
    # Validate that both versions are provided
    if [[ -z "$version1" ]]; then
        echo "[WARNING] compare_versions: version1 is empty or missing" >&2
        return 1
    fi
    
    if [[ -z "$version2" ]]; then
        echo "[WARNING] compare_versions: version2 is empty or missing" >&2
        return 1
    fi
    
    # Use sort -V to compare versions
    # If version2 appears first when sorted, then version1 >= version2
    if [[ "$(printf '%s\n' "$version1" "$version2" | sort -V | head -n1)" == "$version2" ]]; then
        return 0  # version1 >= version2
    else
        return 1  # version1 < version2
    fi
}

# Function to get installed WLP agent version (optimized)
get_installed_wlp_version() {
    local version=""
    
    if is_evergreen_os; then
        # Evergreen: Extract version from container image tag ONLY
        if command -v podman >/dev/null 2>&1; then
            # Get WLP container name (matches: achilles-wlp-wlp.wlp, achilles-wlp-wlp-unstable.wlp-unstable, etc.)
            local container_name=""
            container_name=$(sudo podman ps --format "{{.Names}}" 2>/dev/null | grep -E "achilles-wlp-wlp.*\.wlp" | head -1)
            
            if [[ -n "$container_name" ]]; then
                # Get image name from running container
                local image_name=""
                image_name=$(sudo podman inspect "$container_name" --format '{{.ImageName}}' 2>/dev/null)
                
                # Extract version from image tag (e.g., registry.icm.svc/evergreen/wlp-agent-unstable:0.1.1617 → 0.1.1617)
                if [[ -n "$image_name" ]]; then
                    version=$(echo "$image_name" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)
                fi
            fi
        fi
    else
        # Non-Evergreen: Try binary first, then package managers
        if [[ -z "$version" ]] && [[ -f /opt/wlp-agent/bin/wlp-agent ]]; then
            version=$(/opt/wlp-agent/bin/wlp-agent --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+(-[0-9]+)?' | head -n1)
        fi
        
        # Fallback to RPM-based systems
        if [[ -z "$version" ]] && command -v rpm >/dev/null 2>&1; then
            version=$(rpm -q --queryformat '%{VERSION}' wlp-agent 2>/dev/null)
            # Check if package is not installed
            [[ "$version" == *"not installed"* ]] && version=""
        fi
        
        # Fallback to DEB-based systems
        if [[ -z "$version" ]] && command -v dpkg-query >/dev/null 2>&1; then
            version=$(dpkg-query -W -f='${Version}' wlp-agent 2>/dev/null | cut -d'-' -f1)
        fi
    fi
    
    echo "$version"
}

# Check if running as root or with sudo
if [[ $EUID -eq 0 ]]; then
    echo "[INFO] Running as root"
elif sudo -n true 2>/dev/null; then
    echo "[INFO] Running with sudo access"
else
    echo "[ERROR] This script requires sudo access for comprehensive diagnostics"
    echo "Please run with: sudo $0"
    exit 1
fi 

# Common file paths
WLP_LOG_FILE="/var/log/oracle-cloud-agent/plugins/oci-wlp/oci-wlp.log"
OCA_AGENT_LOG="/var/log/oracle-cloud-agent/agent.log"
WLP_AGENT_LOG="/var/log/wlp-agent/wlp-agent.log"
CVA_UPDATER_LOG="/var/log/cva/cva-updater.log"
INSTANCE_OCID_FILE="/etc/instance-ocid"
OS_RELEASE_FILE="/etc/os-release"
EVERGREEN_RELEASE_FILE="/etc/evergreen-release.json"
SCCP_INSTANCE_FILE="/etc/sccp-instance.json"
CVA_UPDATER_CRON="/etc/cron.d/cva-updater"

# WLP Agent version information
# MAINTAINER NOTE: Update these constants when new WLP versions are released
# Last updated: 2025-09-30
WLP_VERSION="1.1.272-250926"
WLP_MIN_VERSION="1.1.270"
WLP_DOWNLOAD_BASE="https://artifactory.oci.oraclecorp.com:443/wlp-agent-release-generic-local"

# Global variables - detect OS version early
if [[ -f "${OS_RELEASE_FILE}" ]]; then
    os_version=$(grep "^PRETTY_NAME=" "${OS_RELEASE_FILE}" | cut -d= -f2 | tr -d '"')
else
    os_version="Unknown"
fi
ubuntu_version=$(lsb_release -rs 2>/dev/null || echo "")

# Environment and OS image detection functions
is_substrate_environment() {
    # Substrate environment: Chef-managed instances
    # Primary check: sccp-instance.json file exists
    if [[ -f "${SCCP_INSTANCE_FILE}" ]]; then
        return 0
    fi
    
    # Backup check: Use cached substrate metadata service response
    # If the endpoint responded, we're in a substrate environment
    if [[ -n "${SUBSTRATE_METADATA_RESPONSE}" ]]; then
        return 0
    fi
    
    return 1
}

is_evergreen_os() {
    # Evergreen OS: rpm-ostree based immutable system
    # Check for evergreen release file OR rpm-ostree command
    [[ -f "${EVERGREEN_RELEASE_FILE}" ]] || command -v rpm-ostree >/dev/null 2>&1
}

is_ubuntu() {
    # Ubuntu OS: Check os-release file for Ubuntu identification
    # Checks both ID and NAME fields (case-insensitive)
    if [[ -f "${OS_RELEASE_FILE}" ]]; then
        grep -qE "^(ID|NAME)=.*[Uu]buntu" "${OS_RELEASE_FILE}" 2>/dev/null
        return $?
    fi
    return 1
}

# Fetch IMDS metadata once for reuse across multiple sections
IMDS_RESPONSE=$(timeout 10 curl -H "Authorization: Bearer Oracle" -s "http://169.254.169.254/opc/v2/instance/" 2>/dev/null || echo "")

# Fetch substrate metadata once for reuse across multiple sections
SUBSTRATE_METADATA_RESPONSE=$(timeout 10 curl -s "http://instance-metadata.svc/opc/se20170311/instance/" 2>/dev/null || echo "")

section_system_info() {
print_double_header "SYSTEM INFORMATION"

# Get the instance OCID
if [[ -f "${INSTANCE_OCID_FILE}" ]]; then
    if instance_ocid=$(cat "${INSTANCE_OCID_FILE}" 2>/dev/null); then
        echo "Instance OCID: ${instance_ocid}"
    else
        echo "[ISSUE] Cannot read ${INSTANCE_OCID_FILE} file"
    fi
else
    # Use cached IMDS response
    if [[ -n "${IMDS_RESPONSE}" ]]; then
        # Check if the response contains valid JSON and extract 'id'
        if command -v jq >/dev/null 2>&1; then
            if id=$(echo "${IMDS_RESPONSE}" | jq -r '.id' 2>/dev/null) && [[ "${id}" != "null" && -n "${id}" ]]; then
                echo "Instance OCID: ${id}"
            else
                echo "[Warning] Invalid response from OCI metadata service"
            fi
        else
            echo "[WARNING] jq not found - attempting basic parsing"
            # Basic fallback parsing without jq
            if id=$(echo "${IMDS_RESPONSE}" | grep -o '"id":"[^"]*"' | cut -d'"' -f4) && [[ -n "${id}" ]]; then
                echo "Instance OCID: ${id}"
            else
                echo "[Warning] Cannot parse instance OCID without jq"
            fi
        fi
    else    
        # Try sccp-instance.json file first
        if [[ -f "${SCCP_INSTANCE_FILE}" ]] && command -v jq >/dev/null 2>&1; then
            if instance_id=$(jq -r '.instanceId // .instance_id // empty' "${SCCP_INSTANCE_FILE}" 2>/dev/null) && [[ -n "${instance_id}" ]]; then
                echo "Instance ID (from ${SCCP_INSTANCE_FILE}): ${instance_id}"
            fi
        fi
        
        # Try substrate metadata service endpoint as fallback (using cached response)
        if [[ -z "${instance_id:-}" ]] && [[ -n "${SUBSTRATE_METADATA_RESPONSE}" ]]; then
            if command -v jq >/dev/null 2>&1; then
                if instance_id=$(echo "${SUBSTRATE_METADATA_RESPONSE}" | jq -r '.id // .instanceId // empty' 2>/dev/null) && [[ -n "${instance_id}" ]]; then
                    echo "Instance ID (from substrate metadata service): ${instance_id}"
                else
                    echo "[Warning] Could not parse instance ID from substrate metadata service"
                fi
            else
                echo "[WARNING] jq not found - cannot parse substrate metadata service response"
            fi
        fi
        
        # Final error message if all sources failed
        if [[ -z "${instance_id:-}" ]]; then
            echo "[ISSUE] Could not retrieve instance ID from any metadata source"
        fi
    fi
fi

# Get OS version
echo "Operating System Version: ${os_version}"

# Get kernel version
kernel_version=$(uname -r 2>/dev/null || echo "Unknown")
echo "Kernel Version: ${kernel_version}"
}

section_oca_installation_check() {
print_header "OCA Installation Check"
echo "Checking if OCA is running..."

# First check with is-active
if sudo systemctl is-active --quiet oracle-cloud-agent.service; then
    echo "[OK] Oracle Cloud Agent service is active and running"
else
    # Fallback: check with status command and parse output
    status_output=$(sudo systemctl status oracle-cloud-agent.service 2>&1 || true)
    
    # Double-check if status shows active even if is-active failed
    if echo "$status_output" | grep -q "Active: active"; then
        echo "[OK] Oracle Cloud Agent service is active and running (verified via status)"
    else
        echo "[ISSUE] Oracle Cloud Agent service is not running or not installed"
        echo ""
        echo "Service status details:"
        echo "$status_output"
        echo ""
        echo "If you see output like Unit oracle-cloud-agent.service could not be found. It implies that OCA is not installed"
        echo "[ACTION] If your instance is created based on a custom image. Please include OCA into your custom image. You can find latest oca installer in #oca-releases"
        echo "[ACTION] You might want to consult OCA team about how to automate building the latest OCA in your custom image #oci_compute_agent"
        echo "[ACTION] For quick mitigation, you could install OCA on your instances. example oca release: https://dyn.slack.com/archives/C025K5464B1/p1739226415280259 (notice that this link may be obsolete, if so, check #oca-releases for the latest published version)."
    fi
fi
}

section_wlp_plugin_status_check() {
print_header "WLP Plugin Status Check"
echo "Checking if wlp-plugin is running (manages WLP agent lifecycle)..."
if wlp_pid=$(pgrep oci-wlp 2>/dev/null) && [[ -n "${wlp_pid}" ]]; then
    echo "[OK] wlp-plugin is running with PID: ${wlp_pid}"
else
    echo "[ISSUE] wlp-plugin is NOT running"
fi
}

section_wlp_plugin_configuration_check() {
print_header "OCA Plugin Configuration Check"
# Check OCA agent log
echo "Checking OCA agent log: ${OCA_AGENT_LOG}"
if [[ -f "${OCA_AGENT_LOG}" ]] && sudo test -r "${OCA_AGENT_LOG}"; then
    if sudo grep -q "oci-wlp" "${OCA_AGENT_LOG}" 2>/dev/null; then
        echo "[OK] WLP plugin entries found in OCA agent log"
    else
        echo "[ISSUE] No WLP plugin entries found in OCA agent log"
    fi
elif [[ -f "${OCA_AGENT_LOG}" ]]; then
    echo "[ISSUE] OCA agent log file exists but is not readable"
else
    echo "[ISSUE] OCA agent log file not found"
fi
}

section_imds_configuration_check() {
print_header "IMDS Configuration Check"
echo "Checking instance metadata for agent configuration..."
if [[ -n "${IMDS_RESPONSE}" ]]; then
    # Check both conditions using jq (if available) or grep (fallback)
    management_disabled=false
    wlp_disabled=false
    
    if command -v jq >/dev/null 2>&1; then
        # Use jq for both checks ([]? makes iteration optional - handles null without error)
        [ "$(echo "${IMDS_RESPONSE}" | jq -r '.agentConfig.managementDisabled')" = "true" ] && management_disabled=true
        [ "$(echo "${IMDS_RESPONSE}" | jq -r '.agentConfig.pluginsConfig[]? | select(.name=="Cloud Guard Workload Protection") | .desiredState')" = "DISABLED" ] && wlp_disabled=true
    else
        # Use grep for both checks
        echo "${IMDS_RESPONSE}" | grep -q '"managementDisabled": *true' && management_disabled=true
        echo "${IMDS_RESPONSE}" | grep -B2 "Cloud Guard Workload Protection" | grep -q '"desiredState": *"DISABLED"' && wlp_disabled=true
    fi

    if [ "$management_disabled" = "true" ]; then
        echo "[ISSUE] Management plugins are disabled"
        echo ""
        echo "[ACTION] Two ways to enable management plugins:"
        echo ""
        echo "METHOD 1: Via OCI Console"
        echo ""
        echo "METHOD 2: Via UpdateInstance API (CLI)"
        echo "  1. Get current IMDS configuration:"
        echo "     curl -H \"Authorization: Bearer Oracle\" -L http://169.254.169.254/opc/v2/instance/"
        echo ""
        echo "  2. Copy ONLY the \"agentConfig\" object (not the key) to a local file:"
        echo "     # Example: Create file /tmp/agent-config.json with contents like:"
        echo "     https://confluence.oraclecorp.com/confluence/display/TVM/Workload+Protection+Solutions+-+MissingHeartbeat#WorkloadProtectionSolutionsMissingHeartbeat-Managementpluginsaredisabled"
        echo ""
        echo "  3. Update instance configuration:"
        echo "     oci compute instance update --instance-id <instance_ocid> --agent-config file:///tmp/agent-config.json"
        echo ""
        echo "[IMPORTANT] Please File a mitigating CHANGE ticket before making production changes"
        echo "[REFERENCE] https://docs.oracle.com/en-us/iaas/tools/oci-cli/3.66.0/oci_cli_docs/cmdref/compute/instance/update.html"
        echo ""
    fi
    
    if [ "$wlp_disabled" = "true" ]; then
        echo "[ISSUE] WLP plugin explicitly disabled"
        echo ""
        echo "[ACTION] Two ways to enable WLP plugin:"
        echo ""
        echo "METHOD 1: Via OCI Console"
        echo ""
        echo "METHOD 2: Via UpdateInstance API (CLI)"
        echo "  1. Get current IMDS configuration:"
        echo "     curl -H \"Authorization: Bearer Oracle\" -L http://169.254.169.254/opc/v2/instance/"
        echo ""
        echo "  2. Copy ONLY the \"agentConfig\" object to a local file and find the WLP plugin:"
        echo "     # Example: Create file /tmp/agent-config.json and update:"
        echo "     https://confluence.oraclecorp.com/confluence/display/TVM/Workload+Protection+Solutions+-+MissingHeartbeat#WorkloadProtectionSolutionsMissingHeartbeat-WLPPluginisexplicitlydisabled"
        echo ""
        echo "  3. Update instance configuration:"
        echo "     oci compute instance update --instance-id <instance_ocid> --agent-config file:///tmp/agent-config.json"
        echo ""
        echo "[IMPORTANT] File a mitigating CHANGE ticket before making production changes"
        echo "[REFERENCE] https://docs.oracle.com/en-us/iaas/tools/oci-cli/3.66.0/oci_cli_docs/cmdref/compute/instance/update.html"
        echo ""
    fi
    
    if [ "$management_disabled" = "false" ] && [ "$wlp_disabled" = "false" ]; then
        echo "[OK] Instance metadata retrieved successfully - no configuration issues detected"
    fi
else
    echo "[ISSUE] Could not retrieve instance metadata"
fi
}

section_io_timeout_check() {
print_header "I/O Timeout Error Check"
echo "Checking for I/O timeout errors (instance cannot reach WLP dataplane endpoint)..."
wlp_log_file="${WLP_LOG_FILE}"
if [[ -f "${wlp_log_file}" ]] && sudo test -r "${wlp_log_file}"; then
    if [[ -n "${wlp_io_timeout_errors}" ]]; then
        echo "[ISSUE] I/O timeout errors found:"
        echo "Most recent error:"
        echo "${wlp_io_timeout_errors}" | tail -1
        echo ""
        echo "[ACTION] Examine instance networking setup to ensure connectivity"
        echo "[ACTION] Check NAT + Service gateway are in your instance's vcn"
        echo "[ACTION] Check instance subnet's route table has an entry to allow your instance to reach out a public ip"
        echo "[ACTION] Check instance's security list does not block any outbound traffic to a public ip"
        echo "[ACTION] If networking advice doesn't work, contact #oci_vcn team"
    else
        echo "[OK] No I/O timeout errors found"
    fi
elif [[ -f "${wlp_log_file}" ]]; then
    echo "[ISSUE] WLP plugin log exists but is not readable"
else
    echo "[ISSUE] WLP plugin log not found"
fi
}

section_4xx_error_check() {
print_header "4xx Error Check"
echo "Checking WLP logs for 4xx HTTP errors..."

local wlp_log_file="$1"

if [[ ! -f "${wlp_log_file}" ]] || ! sudo test -r "${wlp_log_file}"; then
    echo "[ISSUE] WLP log not found or not readable: ${wlp_log_file}"
    return
fi

# Grep for 4xx errors in real-time (expanded pattern to catch various formats)
local all_4xx_errors=$(sudo grep -E "HTTP/[0-9]\.[0-9] 4[0-9][0-9][^0-9]|Http Status Code: 4[0-9][0-9]|status[: ]+4[0-9][0-9][^0-9]|response[: ]+4[0-9][0-9][^0-9]|POST 4[0-9][0-9][^0-9]|GET 4[0-9][0-9][^0-9]" "${wlp_log_file}" 2>/dev/null || echo "")

# Filter out IMDS v1 related 4xx errors (all 4xx from unauthenticated metadata calls are expected)
# IMDS v1 endpoint: 169.254.169.254/opc/v1/
local errors=$(echo "${all_4xx_errors}" | grep -vE "169\.254\.169\.254.*/opc/v1/" || true)

if [[ -n "${errors}" ]]; then
    echo "[ISSUE] 4xx HTTP errors found in ${wlp_log_file}"
    echo "Recent 4xx errors:"
    echo "${errors}" | tail -3
    echo ""
    
    # Check if this is specifically a resource principal token error with 404/NotAuthorizedOrNotFound (misconfigured target)
    # Check the entire log file (not just the errors variable) for context
    if sudo grep -qE "(NotAuthorizedOrNotFound|404)" "${wlp_log_file}" 2>/dev/null && \
       sudo grep -qE "(getResourcePrincipalToken|Workload Protection is not enabled)" "${wlp_log_file}" 2>/dev/null; then
        echo "[COMMON CAUSE] Misconfigured Target"
        echo ""
        echo "This error indicates that Workload Protection is not enabled for this compartment,"
        echo "or the Cloud Guard Target is not properly configured."
        echo ""
        echo "[ACTION] For manual remediation steps, see:"
        echo "   https://confluence.oraclecorp.com/confluence/display/TVM/Workload+Protection+Solutions+-+MissingHeartbeat#WorkloadProtectionSolutionsMissingHeartbeat-WorkItemforManualRemediation(Configurationissues)"
        echo ""
    fi
else
    echo "[OK] No 4xx HTTP errors found in ${wlp_log_file}"
fi
}

section_dependency_rpms_check() {
print_header "Dependency RPMS Check (Oracle Linux only)"
if echo "${os_version}" | grep -qi "oracle"; then
    echo "Oracle Linux system detected (${os_version}) - checking for dependency errors..."
    wlp_log_file="${WLP_LOG_FILE}"
    if [[ -f "${wlp_log_file}" ]]; then
        if [[ -n "${wlp_dependency_errors}" ]]; then
            echo "[ISSUE] Dependency installation errors found:"
            echo "Most recent error:"
            echo "${wlp_dependency_errors}" | tail -1
            echo ""
            echo "[SOLUTION] Upgrade Oracle Cloud Agent (OCA) to the latest version"
            echo "[INFO] The latest OCA version does NOT require kernel-uek-devel as a prerequisite for WLP agent"
            echo "[INFO] This is the recommended solution to permanently resolve this dependency issue"
            echo ""
            echo "[WORKAROUND] If OCA upgrade is not immediately possible:"
            echo "[ACTION] Run command: sudo yum install kernel-uek-devel -y to confirm if yum works on the instance"
            echo "[ACTION] If yum fails, contact CIO team: #oci-infra-orch"
        else
            echo "[OK] No dependency installation errors found"
        fi
    fi
else
    echo "[INFO] Not Oracle Linux - skipping RPM dependency check"
fi
}

section_sudo_permission_check() {
print_header "Sudo Permission Error Check"
echo "Checking for sudo permission errors..."
wlp_log_file="${WLP_LOG_FILE}"

# Use the global variables set by analyze_wlp_errors
rpm_lock_errors="${wlp_rpm_lock_errors}"
pam_errors="${wlp_pam_errors}"

if [[ -n "${rpm_lock_errors}" ]]; then
    echo "[ISSUE] RPM lock errors found:"
    echo "Most recent error:"
    echo "${rpm_lock_errors}" | tail -1
    echo "[INFO] This means some other process on your host is locking the rpm, or your rpmdb is corrupted"
    echo "[INFO] Sometimes this error occurs on instance startup and goes away"
    echo "[ACTION] If it persists, you need to fix it yourself"
fi

if [[ -n "${pam_errors}" ]]; then
    echo "[ISSUE] PAM account management errors found:"
    echo "Most recent error:"
    echo "${pam_errors}" | tail -1
    echo ""
    echo "[INFO] This means that oracle-cloud-agent user does not have sudo permissions to perform required job"
    echo "[INFO] Chances are high other agents/plugins don't have sudo access either"
    echo ""
    echo "[ACTION] Look for entries related to oracle-cloud-agent and PAM authentication failures"
    echo ""
    print_double_header "SECURE LOG SAMPLE (last 5 entries)"
    sudo tail -5 /var/log/secure 2>/dev/null || echo "Cannot read /var/log/secure"
    echo "[INFO] For full secure log, run: sudo tail -50 /var/log/secure"
    echo ""
    echo "[ACTION] Sudo access may be blocked via access.conf"
    echo "   https://confluence.oraclecorp.com/confluence/display/TVM/Workload+Protection+Solutions+-+MissingHeartbeat#WorkloadProtectionSolutionsMissingHeartbeat-IsWLPplugingettingsudoerrors%3F"
    echo ""
    echo "[INFO] Sudo Policy Enforcement Analysis"
    echo "[INFO] Please figure out what mechanism is being used for sudo access on your fleet:"
    echo "[INFO] 1. sudo is granted via sudoers.d policies"
    echo "[INFO] 2. sudo is granted via ldap (generally configured via a script or chef cookbooks)"
    echo ""
    echo "[ACTION] If first case, check the contents of /etc/sudoers.d/100-oracle-cloud-agent-users"
    echo "[ACTION] In any case, please dont directly edit 100-oracle-cloud-agent sudoers file"
    echo ""
    echo "[ACTION] If second case, fix your scripts or cookbook configurations to allow oracle-cloud-agent user sudo access"
    echo "[INFO] WLP or Oracle Cloud Agent team does not support ldap way of granting sudo access"
    echo ""
    echo "[ACTION] Please reach out to WLP team for further help with below information:"
    echo "[INFO] This info is used to check exact binary path of rpm/systemctl/gpg/dpkg-sig/apt-get/apt"
    echo "[INFO] used by user oracle-cloud-agent is matching that shown in the output of sudo -l -U oracle-cloud-agent"
    echo "[INFO] to verify that no password is required"
    echo ""
    
    if command -v rpm >/dev/null 2>&1; then
        echo "For rpm type systems"
        echo "sudo -u oracle-cloud-agent which rpm"
        echo "$(sudo -u oracle-cloud-agent which rpm 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -u oracle-cloud-agent which systemctl"
        echo "$(sudo -u oracle-cloud-agent which systemctl 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -l -U oracle-cloud-agent"
        sudo -l -U oracle-cloud-agent 2>/dev/null || echo "Cannot check sudo permissions"
        echo ""
        echo "sudo passwd -S oracle-cloud-agent"
        sudo passwd -S oracle-cloud-agent 2>/dev/null || echo "Cannot check password status"
        echo ""
        echo "sudo ls -la /etc/oracle-cloud-agent/plugins/oci-wlp/rpm-gpg-pub-key"
        sudo ls -la /etc/oracle-cloud-agent/plugins/oci-wlp/rpm-gpg-pub-key 2>/dev/null || echo "GPG key file not accessible"
        echo ""
    fi
    
    if command -v apt >/dev/null 2>&1; then
        echo "For ubuntu/debian systems, additional commands"
        echo "sudo -u snap_daemon which gpg"
        echo "$(sudo -u snap_daemon which gpg 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -u snap_daemon which dpkg-sig"
        echo "$(sudo -u snap_daemon which dpkg-sig 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -u snap_daemon which apt-get"
        echo "$(sudo -u snap_daemon which apt-get 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -u snap_daemon which apt"
        echo "$(sudo -u snap_daemon which apt 2>/dev/null || echo 'NOT ACCESSIBLE')"
        echo ""
        echo "sudo -l -U snap_daemon"
        sudo -l -U snap_daemon 2>/dev/null || echo "Cannot check sudo permissions"
        echo ""
        echo "sudo passwd -S snap_daemon"
        sudo passwd -S snap_daemon 2>/dev/null || echo "Cannot check password status"
        echo ""
        echo "sudo ls -la /etc/oracle-cloud-agent/plugins/oci-wlp/rpm-gpg-pub-key"
        sudo ls -la /etc/oracle-cloud-agent/plugins/oci-wlp/rpm-gpg-pub-key 2>/dev/null || echo "GPG key file not accessible"
        echo ""
    fi
fi

if [[ -z "${rpm_lock_errors}" ]] && [[ -z "${pam_errors}" ]]; then
    echo "[OK] No sudo permission errors found"
fi
}

section_rpmdb_corruption_check() {
print_header "RPMDB Corruption Check"
echo "Checking for RPM database corruption..."
wlp_log_file="${WLP_LOG_FILE}"
if [[ -f "${wlp_log_file}" ]]; then
    if [[ -n "${wlp_rpmdb_errors}" ]]; then
        echo "[ISSUE] RPMDB corruption errors detected in logs"
        echo ""
        echo "[ACTION] Check WLP logs for RPMDB corruption errors:"
        echo "sudo grep -i 'rpmdb\|berkeley\|db_runrecovery\|cannot open packages' ${wlp_log_file}"
        echo ""
        echo "[INFO] If wlp-plugin logs show following error:"
        echo ""
        echo "error: rpmdb: BDB0113 Thread/process 2899781/139696647166848 failed: BDB1507 Thread died in Berkeley DB library"
        echo "error: db5 error(-30973) from dbenv->failchk: BDB0087 DB_RUNRECOVERY: Fatal error, run database recovery"
        echo "error: cannot open Packages index using db5 -  (-30973)"
        echo "error: cannot open Packages database in /var/lib/rpm"
        echo ""
        echo "[ISSUE] Your rpmdb is corrupted and needs to be repaired"
        echo ""
        echo "[ACTION] Try the following remediation steps:"
        echo ""
        echo "1. Chef Runbook (for Chef-managed instances):"
        echo "   https://devops.oci.oraclecorp.com/runbooks/CHEF/chef-how-tos/chef-diagnosing-chef-failures/chef-chef-es-doctor-results/err_rpm_corrupt"
        echo ""
        echo "2. Official RPM Database Recovery Guide:"
        echo "   https://rpm.org/user_doc/db_recovery.html"
        echo ""
        echo "[INFO] These guides may not fix all RPMDB corruption issues for all Oracle Linux versions"
        echo "[INFO] If the above solutions don't work, you will need to research and find a solution specific to your system"
        echo ""
        echo "[INFO] Please do not cut a ticket to WLP or OCA team as neither can help with RPMDB corruption"
    else
        echo "[OK] No RPMDB corruption errors found"
    fi
else
    echo "[ISSUE] WLP plugin log file not found - cannot check for RPMDB corruption"
fi
}

section_wlp_agent_minimal_check() {
print_header "WLP Agent Minimal Check"
# Check installed WLP agent version
installed_version=$(get_installed_wlp_version)
if [[ -n "$installed_version" ]]; then
    echo "[INFO] Installed WLP Agent version: ${installed_version}"
    
    # Compare with minimum supported version
    if ! compare_versions "$installed_version" "$WLP_MIN_VERSION"; then
        echo "[WARNING] Agent currently has version ${installed_version}. This does not meet the minimum supported version, ${WLP_MIN_VERSION}"
        echo "[ACTION] Please upgrade WLP agent to version ${WLP_MIN_VERSION} or later"
        echo ""
    fi
else
    echo "[WARNING] Could not determine installed WLP agent version"
fi
}

section_wlp_agent_service_status() {
print_header "WLP Agent Service Status"
echo "Checking if WLP agent service is running..."
if sudo systemctl is-active wlp-agent-osqueryd.service >/dev/null 2>&1; then
    echo "[OK] WLP agent service is running"
else
    echo "[ISSUE] WLP agent service not running or not found"
    echo ""
    
    # Show detailed status for troubleshooting
    if ! sudo systemctl status wlp-agent-osqueryd.service 2>/dev/null; then
        echo "Service status unavailable"
    fi
    echo ""
    echo "[INFO] For complete service logs, run: sudo journalctl -u wlp-agent-osqueryd.service -n 100"
    echo ""
    
    # Check for specific error patterns using journalctl for more comprehensive logs
    service_output=$(sudo journalctl -u wlp-agent-osqueryd.service -n 50 --no-pager 2>&1)
    
    if echo "${service_output}" | grep -q "start-limit"; then
        echo "[ACTION] Service in start-limit state - wait 10-15 minutes or restart service"
        echo "[ACTION] To restart: sudo systemctl restart wlp-agent-osqueryd.service"
        echo "[NOTE] Please file a mitigating CHANGE ticket before restarting in production"
    elif echo "${service_output}" | grep -q "could not be found"; then
        echo "[ACTION] Service unit not found - contact WLP team for investigation"
        echo "[ACTION] For SaaS users on Silver image, consider upgrading to OL8 image"
    else
        echo "[ACTION] Service has other errors - try restarting the service"
        echo "[ACTION] Command: sudo systemctl restart wlp-agent-osqueryd.service"
        echo "[NOTE] Please file a mitigating CHANGE ticket before restarting in production"
    fi
    
    echo ""
    echo "[INFO] If issues persist, create SEV-3 ticket with Workload Protection Service (JIRA-SD queue: WLP)"
    echo "[INFO] Include these logs and details:"
    echo "  - Security Central finding"
    echo "  - TenancyId, compartmentId and instance Id"
    echo ""
    echo "[ACTION] Collect WLP agent logs: /var/log/wlp-agent/wlp-agent.log"
    echo "[ACTION] Collect service logs: sudo journalctl -u wlp-agent-osqueryd.service --no-pager"
    echo "[INFO] For recent logs only: sudo journalctl -u wlp-agent-osqueryd.service -n 100"
fi
}

section_substrate_environment_check() {
    # This function only handles Non-Evergreen Substrate systems
    # Evergreen Substrate systems are handled by section_evergreen_environment_check()
    # Note: This function is only called when substrate environment is already detected
    echo "[INFO] Substrate environment detected - checking CVA-Updater"
    
    # CVA-Updater only works with yum on Non-Evergreen Substrate systems
    local pkg_manager="yum"
    local install_cmd="sudo yum install"
    
    if ! command -v yum >/dev/null 2>&1; then
        echo "[WARNING] yum not found - CVA-Updater requires yum package manager"
        pkg_manager="unknown"
        install_cmd="(yum required for CVA-Updater)"
    fi
    
    print_header "CVA-Updater Installation Check"
    
    # Check if CVA-Updater binary exists
    cva_updater_installed=false
    if sudo test -f "/opt/cva-updater/bin/cva-updater" && rpm -q oci-cva-updater >/dev/null 2>&1; then
        echo "[OK] CVA-Updater is installed and package is present"
        cva_updater_installed=true
    else
        echo "[ISSUE] CVA-Updater is not properly installed"
        
        if sudo test -f "/opt/cva-updater/bin/cva-updater"; then
            echo "  - Binary exists but package missing"
        elif rpm -q oci-cva-updater >/dev/null 2>&1; then
            echo "  - Package installed but binary missing"
            rpm -q oci-cva-updater
        else
            echo "  - Neither binary nor package found"
        fi
        
        show_cva_updater_guidance
    fi
    
    print_header "CVA-Updater Logs Check"
    
    if sudo test -f "${CVA_UPDATER_LOG}" && sudo test -r "${CVA_UPDATER_LOG}"; then
        # Check for recent activity (within last 24 hours) and errors
        recent_activity=false
        has_errors=false
        
        if sudo find "${CVA_UPDATER_LOG}" -mtime -1 2>/dev/null | grep -q "${CVA_UPDATER_LOG}"; then
            recent_activity=true
        fi
        
        if sudo tail -100 "${CVA_UPDATER_LOG}" 2>/dev/null | grep -qi "error\|fail\|exception"; then
            has_errors=true
        fi
        
        if [[ "${recent_activity}" == "true" ]] && [[ "${has_errors}" == "false" ]]; then
            echo "[OK] CVA-Updater logs show recent activity with no errors"
        else
            if [[ "${recent_activity}" == "false" ]]; then
                echo "[WARNING] CVA-Updater log has not been modified recently"
                echo "[INFO] CVA-Updater should run every hour"
            fi
            
            if [[ "${has_errors}" == "true" ]]; then
                echo "[ISSUE] Errors detected in recent CVA-Updater logs"
                echo "Recent errors:"
                sudo tail -100 "${CVA_UPDATER_LOG}" 2>/dev/null | grep -i "error\|fail\|exception" | tail -3
                echo ""
                echo "[ACTION] Check CVA-Updater logs for details: sudo tail -100 ${CVA_UPDATER_LOG}"
                echo "[ACTION] Common causes: yum repository issues, network problems, package conflicts"
            fi
        fi
        
        echo "[INFO] To analyze CVA-Updater logs: sudo tail -50 ${CVA_UPDATER_LOG}"
    else
        echo "[ISSUE] CVA-Updater log file not found or not readable: ${CVA_UPDATER_LOG}"
        echo "[INFO] This could indicate CVA-Updater is not running or has never run"
        show_cva_updater_guidance
    fi
    
    print_header "CVA-Updater Cron Configuration Check"
    
    # Check if CVA-Updater cron file exists
    if sudo test -f "${CVA_UPDATER_CRON}"; then
        # Check cron content
        if cron_content=$(sudo cat "${CVA_UPDATER_CRON}" 2>/dev/null); then
            # Check if actual cron command line is commented out
            if echo "${cron_content}" | grep -q "^#.*\*.*\*.*\*.*\*.*\*.*cva-updater"; then
                echo "[ISSUE] CVA-Updater cron entries are commented out"
                echo "[ACTION] Uncomment the cva-updater cron entries to enable automatic execution"
                echo "[INFO] To view cron configuration: sudo cat ${CVA_UPDATER_CRON}"
                show_cva_updater_guidance
            elif echo "${cron_content}" | grep -q "^\*.*\*.*\*.*\*.*\*.*cva-updater\|^[0-9].*\*.*\*.*\*.*\*.*cva-updater"; then
                echo "[OK] CVA-Updater cron is properly configured"
            else
                echo "[WARNING] CVA-Updater cron configuration format not recognized"
                echo "[ACTION] Verify cron entries are properly formatted and not commented out"
                echo "[INFO] To view cron configuration: sudo cat ${CVA_UPDATER_CRON}"
            fi
        else
            echo "[ISSUE] Cannot read cva-updater cron file"
            show_cva_updater_guidance
        fi
    else
        echo "[ISSUE] CVA-Updater cron file not found: ${CVA_UPDATER_CRON}"
        echo "[ACTION] CVA-Updater cron job is required for automatic WLP agent updates in substrate environments"
        show_cva_updater_guidance
    fi
    
    # Check if we need to verify chef status (only when cva-updater is not installed on non-EVG substrate)
    if [[ "${cva_updater_installed}" == "false" ]] && [[ ! -f "${EVERGREEN_RELEASE_FILE}" ]]; then
        print_header "Chef Status Check"
        
        chef_history_file="/etc/chef/.chef-history-status"
        if [[ -f "${chef_history_file}" ]] && sudo test -r "${chef_history_file}"; then
            # Check recent activity and success/failure
            recent_chef=false
            chef_success=false
            
            if sudo find "${chef_history_file}" -mtime -1 2>/dev/null | grep -q "${chef_history_file}"; then
                recent_chef=true
            fi
            
            if grep -qi "success\|complete" "${chef_history_file}" 2>/dev/null; then
                chef_success=true
            fi
            
            if [[ "${recent_chef}" == "true" ]] && [[ "${chef_success}" == "true" ]]; then
                echo "[OK] Chef is running properly with recent successful activity"
            else
                if [[ "${recent_chef}" == "false" ]]; then
                    echo "[WARNING] Chef has not run recently (no activity within 24 hours)"
                fi
                
                if [[ "${chef_success}" == "false" ]]; then
                    if grep -qi "fail\|error" "${chef_history_file}" 2>/dev/null; then
                        echo "[ISSUE] Chef history indicates failures"
                        echo "[ACTION] Chef failures may prevent CVA-Updater installation"
                        echo "[ACTION] Run Chef doctor script to diagnose: https://devops.oci.oraclecorp.com/runbooks/CHEF/chef-how-tos/chef-diagnosing-chef-failures/chef-how-to-use-chef-doctor"
                    else
                        echo "[WARNING] Cannot determine chef success status"
                    fi
                fi
                
                echo "[ACTION] Review chef logs: sudo journalctl -u chef-client"
            fi
            
            echo "[INFO] To view chef status: cat ${chef_history_file}"
        else
            echo "[ISSUE] Chef history status file not found: ${chef_history_file}"
            echo "[ACTION] Verify Chef is properly configured and running"
            echo "[ACTION] Run Chef doctor script to diagnose: https://devops.oci.oraclecorp.com/runbooks/CHEF/chef-how-tos/chef-diagnosing-chef-failures/chef-how-to-use-chef-doctor"
            echo "[ACTION] Check chef client logs: sudo journalctl -u chef-client"
        fi
    fi
    
    print_header "WLP Agent Package Availability"
    
    # Substrate environments use yum/dnf for CVA-Updater
    # (Evergreen systems with rpm-ostree are handled by section_evergreen_environment_check)
    if [[ "${pkg_manager}" == "yum" ]]; then
        # Try rpm first (faster and more reliable in scripts)
        if rpm -q wlp-agent >/dev/null 2>&1; then
            echo "[OK] WLP agent package is installed"
            installed_version=$(rpm -q --queryformat '%{VERSION}-%{RELEASE}' wlp-agent 2>/dev/null)
            if [[ -n "${installed_version}" ]]; then
                echo "[INFO] Installed version: ${installed_version}"
            fi
            echo "[INFO] To check for updates: yum info --showduplicates wlp-agent"
        else
            # Fallback to yum if rpm doesn't find it (package might be in repos but not installed)
            if yum info wlp-agent >/dev/null 2>&1; then
                echo "[WARNING] WLP agent package is available in repositories but NOT installed"
                echo "[ACTION] Install the package: sudo yum install -y wlp-agent"
            else
                echo "[ISSUE] WLP agent package not found in rpm database or yum repositories"
                echo "[INFO] To check manually: rpm -q wlp-agent or yum info wlp-agent"
                show_cva_updater_guidance
            fi
        fi
    else
        echo "[WARNING] yum not available - CVA-Updater requires yum package manager"
        echo "[INFO] Current package manager: ${pkg_manager}"
        show_cva_updater_guidance
    fi
}

section_evergreen_environment_check() {
    # This function handles ALL Evergreen systems (both Overlay and Substrate)
    # Note: This function is only called when evergreen OS is already detected
    
    print_header "Is WLP agent container there?"
    if container_check=$(sudo podman ps --format "{{.Names}}" 2>/dev/null | grep -E "achilles-wlp-wlp\.wlp") && [[ -n "${container_check}" ]]; then
        echo "[OK] WLP agent container found"
    else
        echo "[ISSUE] WLP agent container NOT found"
        echo ""
        echo "[INFO] Use below commands to check if WLP agent is there. If WLP agent is not found, then contact"
        echo "[INFO] Evergreen team(#oci_evergreen_onboard) about how to fix your instance to have WLP agent installed"
        echo "[INFO] (e.g. use a newer EVG VM image to reprovision your instance)."
        echo ""
        echo "[NOTE] Please also check if there is any os patching failure for your Evergreen instances."
        echo "[NOTE] If so, fix these os patching and see if that help bring back WLP agent."
        echo ""
        echo "[INFO] For any questions if achilles is not working please reach out to the evergreen/ticm team"
        echo ""
        echo "sudo podman ps | grep achilles-wlp-wlp.wlp"
    fi
    
    print_header "Is achilles working?"
    if sudo systemctl status achilles-evergreen-achilles.service >/dev/null 2>&1; then
        echo "[OK] Achilles service is running"
    else
        echo "[ISSUE] Achilles service not working properly"
        echo "Service status check failed"
        echo ""
        echo "[INFO] If WLP agent is on older version, check the status of achilles service"
        echo ""
        echo "sudo systemctl status achilles-evergreen-achilles.service"
        echo ""
        echo "[ACTION] Setting up ticm-doctor to diagnose Achilles/Evergreen issues..."
        echo ""
        
        # Check if ticm-doctor is already installed
        ticm_doctor_available=false
        if command -v ticm-doctor >/dev/null 2>&1; then
            echo "[OK] ticm-doctor found at: $(command -v ticm-doctor)"
            ticm_doctor_available=true
        else
            echo "[INFO] Attempting to download ticm-doctor..."
            # Try to download ticm-doctor from Bitbucket
            if sudo curl -sSL -o /usr/local/bin/ticm-doctor \
                https://bitbucket.oci.oraclecorp.com/projects/FORESTRY/repos/achilles/raw/scripts/ticm-doctor 2>/dev/null && \
               [[ -f /usr/local/bin/ticm-doctor ]]; then
                sudo chmod +x /usr/local/bin/ticm-doctor
                echo "[OK] ticm-doctor installed successfully at /usr/local/bin/ticm-doctor"
                ticm_doctor_available=true

            else
                echo "[WARNING] Failed to download ticm-doctor (network/firewall/auth issue)"
                echo ""
                echo "[ACTION] Please install ticm-doctor manually:"
                echo ""
                echo "1. Get the ticm-doctor script from Bitbucket:"
                echo "   https://bitbucket.oci.oraclecorp.com/projects/FORESTRY/repos/achilles/browse/scripts/ticm-doctor"
                echo ""
                echo "2. Save the script to /usr/local/bin/ticm-doctor on this host"
                echo ""
                echo "3. Make it executable and run it:"
                echo "   # sudo chmod +x /usr/local/bin/ticm-doctor"
                echo "   # sudo ticm-doctor"
                echo ""
                echo "[INFO] Upload the generated ticm-doctor-*.tar file for further analysis"
            fi
        fi
        
        # Run ticm-doctor if available
        if [[ "${ticm_doctor_available}" == "true" ]]; then
            echo ""
            echo "[INFO] Running ticm-doctor..."
            echo ""
            sudo ticm-doctor || echo "[WARNING] ticm-doctor execution failed"
        fi
        echo ""
    fi
    
    print_header "Is achilles successfully deployed WLP agent to the underlying instance?"
    echo "[INFO] Collecting achilles deployment log for WLP agent..."
    echo ""
    # Collect the agent deployment log directly to OUTPUT_DIR
    agent_deployment_log="${OUTPUT_DIR}/agent-deployment.log"
    if sudo journalctl -u achilles-evergreen-achilles.service -g 'wlp/wlp' > "${agent_deployment_log}" 2>/dev/null; then
        if deployment_logs=$(tail -5 "${agent_deployment_log}") && [[ -n "${deployment_logs}" ]]; then
            echo "[OK] Recent deployment log entries found"
        fi
    else
        echo "[ISSUE] No deployment logs found"
    fi
    
    print_header "Is achilles having issues with WLP container?"
    echo "[INFO] Collect achilles log for WLP agent with below command"
    echo ""
    echo "sudo journalctl -u achilles-evergreen-achilles.service -o cat -f | grep agent-wlp"
    echo ""
    if wlp_logs=$(sudo journalctl -u achilles-evergreen-achilles.service -o cat 2>/dev/null | grep -i wlp | tail -3) && [[ -n "${wlp_logs}" ]]; then
        echo "[OK] Logs found"
        echo ""
        echo "[INFO] For full logs, run: sudo journalctl -u achilles-evergreen-achilles.service -o cat | grep -i wlp"
    else
        echo "[ISSUE] No WLP-related logs found"
    fi
    
    print_header "Is WLP agent systemd service there?"
    echo "Checking WLP agent systemd service status..."
    echo ""
    service_output=$(sudo systemctl status wlp-agent-osqueryd.service 2>&1)
    if echo "${service_output}" | grep -q "could not be found"; then
        echo "Unit wlp-agent-osqueryd.service could not be found"
        echo "[ISSUE] WLP agent systemd service not found"
        echo "[INFO] Use below commands to check if WLP agent systemd is there."
        echo "sudo systemctl status wlp-agent-osqueryd.service"
        if [[ -n "${container_check:-}" ]]; then
            echo "[ACTION] Container exists but systemd service missing - recreate container"
            echo ""
            echo "[INFO] If WLP agent container is there but systemd service is missing, please run command"
            echo "[INFO] sudo podman rm -f achilles-wlp-wlp.wlp to recreate the container. If the issue happens"
            echo "[INFO] repeatedly, please engage WLP team to debug further."
        fi
    else
        if sudo systemctl status wlp-agent-osqueryd.service >/dev/null 2>&1; then
            echo "[OK] WLP agent systemd service found"
        else
            sudo systemctl status wlp-agent-osqueryd.service 2>/dev/null
            echo "[WARNING] WLP agent systemd service check failed"
        fi
        echo ""
        echo "[INFO] For complete service logs, run: sudo journalctl -u wlp-agent-osqueryd.service -n 100"
    fi
}

section_ubuntu_environment() {
print_header "Ubuntu Environment (20/22/23/24)"
echo "Checking for Ubuntu environment..."

# Detect Ubuntu version
ubuntu_major_version=""
if [[ "${ubuntu_version}" =~ ^([0-9]+)\. ]]; then
    ubuntu_major_version="${BASH_REMATCH[1]}"
else
    # Extract version from os-release if lsb_release didn't work
    ubuntu_version_from_file=$(grep "VERSION_ID=" "${OS_RELEASE_FILE}" 2>/dev/null | cut -d= -f2 | tr -d '"')
    if [[ "${ubuntu_version_from_file}" =~ ^([0-9]+)\. ]]; then
        ubuntu_major_version="${BASH_REMATCH[1]}"
        ubuntu_version="${ubuntu_version_from_file}"
    fi
fi

if [[ "${ubuntu_major_version}" == "20" ]] || [[ "${ubuntu_major_version}" == "22" ]] || [[ "${ubuntu_major_version}" == "23" ]] || [[ "${ubuntu_major_version}" == "24" ]]; then
    echo "[OK] Ubuntu ${ubuntu_major_version} detected (version: ${ubuntu_version})"
    
    # Version-specific support information
    case "${ubuntu_major_version}" in
        "20")
            echo "[INFO] Ubuntu 20 - WLP agent supported with compatible packages"
            ;;
        "22")
            echo "[INFO] Ubuntu 22 - WLP agent supported with compatible packages"
            ;;
        "23")
            echo "[INFO] Ubuntu 23 - WLP agent supported with compatible packages"
            ;;
        "24")
            echo "[INFO] Ubuntu 24 - WLP agent supported from version 1.1.244"
            ;;
    esac
    
    if sudo systemctl is-active wlp-agent-osqueryd.service >/dev/null 2>&1; then
        echo "[OK] WLP agent service is running"
    else
        echo ""
        echo "[INFO] To check WLP agent status, run: sudo systemctl status wlp-agent-osqueryd.service"
        echo "[INFO] For complete service logs, run: sudo journalctl -u wlp-agent-osqueryd.service -n 100"
        echo ""
        echo "[INFO] Expected output when WLP agent is running correctly:"
        echo "● wlp-agent-osqueryd.service - The osquery Daemon"
        echo "     Loaded: loaded (/usr/lib/systemd/system/wlp-agent-osqueryd.service; enabled; preset: enabled)"
        echo "    Drop-In: /etc/systemd/system.control/wlp-agent-osqueryd.service.d"
        echo "             └─50-CPUQuota.conf, 50-MemoryMax.conf"
        echo "     Active: active (running) since Wed 2025-02-26 06:11:03 UTC; 1 week 6 days ago"
        echo "   Main PID: 33357 (osqueryd)"
        echo "      Tasks: 26 (limit: 19126)"
        echo "     Memory: 65.1M (max: 600.0M limit: 250.0M available: 534.8M peak: 83.2M)"
        echo "        CPU: 46min 43.682s"
        echo "     CGroup: /system.slice/wlp-agent-osqueryd.service"
        echo "             ├─33357 /opt/wlp-agent/bin/osqueryd --flagfile /opt/wlp-agent/configs/osquery.flags"
        echo "             ├─33359 /opt/wlp-agent/bin/osqueryd"
        echo "             └─33360 /opt/wlp-agent/extensions/wlp-agent.ext --socket /var/wlp-agent/osquery.em --timeout 3600 --interval 3"
        echo ""
        echo "[ISSUE] WLP agent not installed on Ubuntu ${ubuntu_major_version}"
        echo "[ACTION] Download and install manually:"
        
        # Version-specific download links
        case "${ubuntu_major_version}" in
            "20"|"22"|"23")
                echo "AMD: ${WLP_DOWNLOAD_BASE}/wlp-agent-${WLP_VERSION}.ubuntu1.amd64.deb"
                echo "ARM: ${WLP_DOWNLOAD_BASE}/wlp-agent-${WLP_VERSION}.ubuntu1.arm64.deb"
                ;;
            "24")
                echo "AMD: ${WLP_DOWNLOAD_BASE}/wlp-agent-${WLP_VERSION}.ubuntu24.amd64.deb"
                echo "ARM: ${WLP_DOWNLOAD_BASE}/wlp-agent-${WLP_VERSION}.ubuntu24.arm64.deb"
                ;;
        esac
        
        echo ""
        echo "[ACTION] Once you have downloaded the installer, you can scp it to your target instance:"
        echo "scp <your-installer-local-path> ubuntu@<remote-instance>:<your-installer-remote-path>"
        echo ""
        echo "[ACTION] After the installer is copied to your instance, execute the below command to install it:"
        echo "sudo dpkg -i <your-installer-full-path>"
    fi
else
    echo "[INFO] Ubuntu detected but not a supported version (20/22/23/24)"
    if [[ -n "${ubuntu_version}" ]]; then
        echo "[INFO] Detected version: ${ubuntu_version}"
    else
        echo "[INFO] Could not determine Ubuntu version"
    fi
    echo "[INFO] WLP agent may still work on this Ubuntu version"
fi
}

# Main diagnostic execution function
# Detects environment and OS image type, then runs appropriate diagnostic sections
# Environment types: overlay (standard OCI) or substrate (Chef-managed)
# OS image types: non-evergreen (mutable) or evergreen (immutable/rpm-ostree)
run_diagnostics() {
    # Always run system info first
    section_system_info
    
    # Detect environment and OS image type
    local environment=""
    local os_image=""
    
    if is_substrate_environment; then
        environment="substrate"
    else
        environment="overlay"
    fi
    
    if is_evergreen_os; then
        os_image="evergreen"
    else
        os_image="non-evergreen"
    fi
    
    print_double_header "DETECTED CONFIGURATION"
    echo "[INFO] Environment: ${environment^}"
    echo "[INFO] OS Image: ${os_image^}"
    echo "[INFO] Combination: ${os_image^} ${environment^}"
    section_wlp_agent_minimal_check
    # ========================================================================
    # DIAGNOSTIC SECTION EXECUTION BASED ON DETECTED ENVIRONMENT
    # ========================================================================
    # 
    # WLP Agent Management by Environment:
    #   - Non-Evergreen Overlay: Managed by OCA WLP Plugin
    #   - Non-Evergreen Substrate: Managed by CVA-Updater
    #   - Evergreen (both Overlay and Substrate): Managed by Achilles agent
    # ========================================================================
    
    # NON-EVERGREEN OVERLAY: Run OCA WLP Plugin diagnostics
    # WLP agent is managed by OCA WLP Plugin (not CVA-Updater)
    # Checks: OCA service, WLP plugin, IMDS config, network errors, permissions
    if [[ "$environment" == "overlay" ]] && [[ "$os_image" == "non-evergreen" ]]; then
        analyze_wlp_linux_logs "${WLP_LOG_FILE}"
        section_oca_installation_check
        section_wlp_plugin_status_check
        section_wlp_plugin_configuration_check
        section_imds_configuration_check
        section_io_timeout_check
        section_4xx_error_check "${WLP_LOG_FILE}"
        section_dependency_rpms_check
        section_sudo_permission_check
        section_rpmdb_corruption_check
        section_wlp_agent_service_status
    fi
    
    # NON-EVERGREEN SUBSTRATE: Run CVA-Updater diagnostics
    # WLP agent is managed by CVA-Updater (not OCA WLP Plugin)
    # Checks: CVA-Updater installation, logs, cron, package availability
    if [[ "$environment" == "substrate" ]] && [[ "$os_image" == "non-evergreen" ]]; then
        section_substrate_environment_check
        section_4xx_error_check "${WLP_AGENT_LOG}"
    fi
    
    # EVERGREEN OS: Run container-based WLP diagnostics
    # WLP agent is managed by Achilles agent (applies to both Overlay and Substrate)
    # Checks: WLP container, Achilles service, container deployment, systemd service, 4xx errors
    if [[ "$os_image" == "evergreen" ]]; then
        section_evergreen_environment_check
        section_4xx_error_check "${WLP_AGENT_LOG}"
    fi
    
    # UBUNTU: Run Ubuntu-specific WLP diagnostics
    # Checks: Ubuntu version support, WLP manual installation status
    # Applies to all environment combinations (20/22/23/24)
    # Only run if this is actually Ubuntu (not Oracle Linux, RHEL, etc.)
    if is_ubuntu; then
        section_ubuntu_environment
    fi
}

# Run comprehensive diagnostic based on detected components
run_diagnostics

# Simple file collection and tar creation (only log errors)
# Copy log files if they exist with renamed filenames
declare -A log_files=(
    ["${WLP_AGENT_LOG}"]="wlp-agent.log"
    ["${WLP_LOG_FILE}"]="oci-wlp.log"
    ["${OCA_AGENT_LOG}"]="oca-agent.log"
    ["${CVA_UPDATER_LOG}"]="cva-updater.log"
)

for source in "${!log_files[@]}"; do
    if [[ -f "${source}" ]]; then
        if ! sudo cp "${source}" "${OUTPUT_DIR}/${log_files[$source]}" 2>/dev/null; then
            echo "[ISSUE] Failed to copy log file: ${source}"
        fi
    fi
done

# Copy config files if they exist
for file in "${INSTANCE_OCID_FILE}" "${OS_RELEASE_FILE}"; do
    if [[ -f "$file" ]]; then
        if ! cp "$file" "${OUTPUT_DIR}/" 2>/dev/null; then
            echo "[ISSUE] Failed to copy config file: ${file}"
        fi
    fi
done

# Create tar archive in the same parent directory as OUTPUT_DIR
OUTPUT_PARENT_DIR=$(dirname "${OUTPUT_DIR}")
OUTPUT_BASENAME=$(basename "${OUTPUT_DIR}")
ARCHIVE_PATH="${OUTPUT_PARENT_DIR}/${OUTPUT_BASENAME}.tar.gz"

cd "${OUTPUT_PARENT_DIR}" || exit 1
tar -czf "${OUTPUT_BASENAME}.tar.gz" "${OUTPUT_BASENAME}/" 2>/dev/null

print_double_header "DOCUMENTATION & SUPPORT"
echo "[REFERENCE] WLP Missing Heartbeat Solution Guide:"
echo "  https://confluence.oraclecorp.com/confluence/display/TVM/Workload+Protection+Solutions+-+MissingHeartbeat"

# Display summary of all detected issues at the very end
print_double_header "SUMMARY OF DETECTED ISSUES"


# Extract all [ISSUE] lines with their section context from the diagnosis log
# Find section headers (lines between dashes) and associate with issues
issue_sections=$(awk '
    /^---------------------------------------$/ { getline; if ($0 !~ /^---/) { section=$0; getline } next }
    /\[ISSUE\]/ { print section ": " $0 }
' "${OUTPUT_DIR}/diagnosis_output.log" 2>/dev/null || echo "")

if [[ -z "$issue_sections" ]]; then
    echo "[OK] No issues detected!"
    echo ""
else
    echo "Issues found during diagnosis. Please review the following sections above:"
    
    # Get unique section names that have issues
    counter=1
    echo "$issue_sections" | while IFS= read -r line; do
        section_name=$(echo "$line" | cut -d':' -f1)
        issue_text=$(echo "$line" | sed 's/^[^:]*: \[ISSUE\] //')
        print_header "${counter}. ${section_name}"
        echo "      Issue: ${issue_text}"
        ((counter++))
    done
    
    echo ""
    echo "Each section above contains [ACTION] items with specific remediation steps."
    print_double_header "STILL NEED HELP?"
    echo "If you have reviewed the sections above and still need assistance:"
    echo "  1. Collect the diagnosis archive: ${ARCHIVE_PATH}"
    echo "  2. Reach out to the WLP team via Slack: #oci_workloadprotection_users"
fi
echo ""