plot_resource_tracker.py

"""
 Copyright (C) 2025 Intel Corporation

 SPDX-License-Identifier: MIT

"""
#!/usr/bin/env python3
"""
Plot system resource tracking data from Level Zero resource tracker CSV output.

Usage:
    python3 plot_resource_tracker.py <csv_file>

Example:
    export ZEL_ENABLE_SYSTEM_RESOURCE_TRACKER_CHECKER=1
    export ZEL_SYSTEM_RESOURCE_TRACKER_CSV=tracker_output.csv
    export ZEL_ENABLE_LOADER_LOGGING=1
    export ZEL_LOADER_LOGGING_LEVEL=debug
    ./my_level_zero_app
    python3 plot_resource_tracker.py tracker_output.csv
"""

import sys
import pandas as pd
import matplotlib.pyplot as plt
from pathlib import Path

def plot_resource_tracker(csv_file):
    """Plot resource tracking data from CSV file."""
    
    # Read CSV file
    df = pd.read_csv(csv_file)
    
    # Check if CSV has any data rows
    if len(df) == 0:
        print(f"Error: CSV file '{csv_file}' contains no data rows (only header).")
        print("Make sure the application runs with ZEL_ENABLE_SYSTEM_RESOURCE_TRACKER_CHECKER=1")
        return
    
    # Convert time from milliseconds to seconds
    df['TimeSec'] = df['TimeMs'] / 1000.0
    
    # Detect potential memory leaks - track memory increases that don't get freed
    # For create/destroy operations, memory should return to baseline
    # Look for APIs that destroy resources but memory doesn't decrease
    df['PotentialLeak_KB'] = 0
    
    # Create/Destroy pairs to monitor for leaks
    create_destroy_pairs = {
        'zeContextCreate': 'zeContextDestroy',
        'zeCommandQueueCreate': 'zeCommandQueueDestroy',
        'zeModuleCreate': 'zeModuleDestroy',
        'zeKernelCreate': 'zeKernelDestroy',
        'zeEventPoolCreate': 'zeEventPoolDestroy',
        'zeCommandListCreate': 'zeCommandListDestroy',
        'zeCommandListCreateImmediate': 'zeCommandListDestroy',
        'zeEventCreate': 'zeEventDestroy',
        'zeFenceCreate': 'zeFenceDestroy',
        'zeImageCreate': 'zeImageDestroy',
        'zeSamplerCreate': 'zeSamplerDestroy',
        'zeMemAllocDevice': 'zeMemFree',
        'zeMemAllocHost': 'zeMemFree',
        'zeMemAllocShared': 'zeMemFree',
    }
    
    # Track memory at create and check if it decreased at destroy
    for idx, row in df.iterrows():
        api = row['APICall']
        # If this is a destroy operation, check if memory decreased
        if api in create_destroy_pairs.values():
            # Memory should decrease on destroy - if it increased, it's a leak
            if row['Delta_VmRSS_KB'] > 100:  # Threshold: 100KB increase on destroy = leak
                df.at[idx, 'PotentialLeak_KB'] = row['Delta_VmRSS_KB']
    
    # Calculate cumulative leaks over time
    df['CumulativeLeak_KB'] = df['PotentialLeak_KB'].cumsum()
    
    # Create figure with multiple subplots
    fig, axes = plt.subplots(4, 3, figsize=(18, 16))
    fig.suptitle(f'Level Zero System Resource Tracking\n{Path(csv_file).name}', fontsize=16)
    
    # Plot 1: Memory Leak Detection Over Time
    ax1 = axes[0, 0]
    # Plot cumulative leaks
    ax1.plot(df['TimeSec'], df['CumulativeLeak_KB'] / 1024, label='Cumulative Leaks', 
             linewidth=3, color='red')
    # Mark individual leak events
    leak_events = df[df['PotentialLeak_KB'] > 0]
    if not leak_events.empty:
        ax1.scatter(leak_events['TimeSec'], leak_events['CumulativeLeak_KB'] / 1024, 
                   color='darkred', s=100, marker='x', linewidths=3, 
                   label=f'Leak Events ({len(leak_events)})', zorder=5)
    ax1.set_xlabel('Time (s)')
    ax1.set_ylabel('Memory Leaked (MB)')
    ax1.set_title('Memory Leak Detection Over Time', fontweight='bold', color='darkred')
    ax1.legend()
    ax1.grid(True, alpha=0.3)
    if df['CumulativeLeak_KB'].max() > 0:
        ax1.set_facecolor('#fff5f5')  # Light red background if leaks detected
    
    # Plot 2: Memory Usage Over Time (VmRSS, VmSize, VmData)
    ax2 = axes[0, 1]
    ax2.plot(df['TimeSec'], df['VmRSS_KB'] / 1024, label='VmRSS', linewidth=2)
    ax2.plot(df['TimeSec'], df['VmSize_KB'] / 1024, label='VmSize', linewidth=2, alpha=0.7)
    ax2.plot(df['TimeSec'], df['VmData_KB'] / 1024, label='VmData', linewidth=2, alpha=0.7)
    ax2.set_xlabel('Time (s)')
    ax2.set_ylabel('Memory (MB)')
    ax2.set_title('System Memory Usage Over Time')
    ax2.legend()
    ax2.grid(True, alpha=0.3)
    
    # Plot 3: Leak Events by API Type
    ax3 = axes[0, 2]
    if not leak_events.empty:
        leak_by_api = leak_events.groupby('APICall')['PotentialLeak_KB'].sum().sort_values(ascending=True) / 1024
        if not leak_by_api.empty:
            leak_by_api.plot(kind='barh', ax=ax3, color='orangered')
            ax3.set_xlabel('Total Leaked Memory (MB)')
            ax3.set_title('Memory Leaks by API Call', fontweight='bold')
            ax3.grid(True, alpha=0.3, axis='x')
            ax3.set_facecolor('#fff5f5')
        else:
            ax3.text(0.5, 0.5, 'No Leaks Detected!', ha='center', va='center', 
                    fontsize=14, color='green', fontweight='bold', transform=ax3.transAxes)
            ax3.set_title('Memory Leaks by API Call')
            ax3.axis('off')
    else:
        ax3.text(0.5, 0.5, 'No Leaks Detected!', ha='center', va='center', 
                fontsize=14, color='green', fontweight='bold', transform=ax3.transAxes)
        ax3.set_title('Memory Leaks by API Call')
        ax3.axis('off')
    
    # Plot 4: Memory Deltas (showing per-call changes)
    ax4 = axes[1, 0]
    ax4.plot(df['TimeSec'], df['Delta_VmRSS_KB'] / 1024, label='Delta VmRSS', linewidth=1.5)
    ax4.plot(df['TimeSec'], df['Delta_VmSize_KB'] / 1024, label='Delta VmSize', linewidth=1.5, alpha=0.7)
    ax4.axhline(y=0, color='black', linestyle='--', linewidth=0.5)
    ax4.set_xlabel('Time (s)')
    ax4.set_ylabel('Memory Change (MB)')
    ax4.set_title('Memory Deltas Per API Call')
    ax4.legend()
    ax4.grid(True, alpha=0.3)
    
    # Plot 5: Level Zero Resource Counts
    ax5 = axes[1, 1]
    has_resources = False
    if df['Contexts'].max() > 0:
        ax5.plot(df['TimeSec'], df['Contexts'], label='Contexts', linewidth=2)
        has_resources = True
    if df['CommandQueues'].max() > 0:
        ax5.plot(df['TimeSec'], df['CommandQueues'], label='CommandQueues', linewidth=2)
        has_resources = True
    if df['Modules'].max() > 0:
        ax5.plot(df['TimeSec'], df['Modules'], label='Modules', linewidth=2)
        has_resources = True
    if df['Kernels'].max() > 0:
        ax5.plot(df['TimeSec'], df['Kernels'], label='Kernels', linewidth=2)
        has_resources = True
    ax5.set_xlabel('Time (s)')
    ax5.set_ylabel('Resource Count')
    ax5.set_title('L0 Resource Counts (Contexts, Queues, Modules, Kernels)')
    if has_resources:
        ax5.legend()
    ax5.grid(True, alpha=0.3)
    
    # Plot 6: Command Lists and Event Resources
    ax6 = axes[1, 2]
    has_cmd_resources = False
    if df['CommandLists'].max() > 0:
        ax6.plot(df['TimeSec'], df['CommandLists'], label='CommandLists', linewidth=2)
        has_cmd_resources = True
    if df['EventPools'].max() > 0:
        ax6.plot(df['TimeSec'], df['EventPools'], label='EventPools', linewidth=2)
        has_cmd_resources = True
    if df['Events'].max() > 0:
        ax6.plot(df['TimeSec'], df['Events'], label='Events', linewidth=2)
        has_cmd_resources = True
    if df['Fences'].max() > 0:
        ax6.plot(df['TimeSec'], df['Fences'], label='Fences', linewidth=2)
        has_cmd_resources = True
    ax6.set_xlabel('Time (s)')
    ax6.set_ylabel('Resource Count')
    ax6.set_title('L0 Command Lists and Events')
    if has_cmd_resources:
        ax6.legend()
    ax6.grid(True, alpha=0.3)
    
    # Plot 7: Total Memory Allocations
    ax7 = axes[2, 0]
    ax7.plot(df['TimeSec'], df['TotalMemory_Bytes'] / (1024*1024), label='Total Memory', 
             linewidth=2, color='red')
    ax7.set_xlabel('Time (s)')
    ax7.set_ylabel('Memory (MB)')
    ax7.set_title('Total L0 Memory Allocations')
    ax7.legend()
    ax7.grid(True, alpha=0.3)
    
    # Plot 8: API Call Distribution (top 10 most frequent)
    ax8 = axes[2, 1]
    api_counts = df['APICall'].value_counts().head(10).sort_values(ascending=True)
    if len(api_counts) > 0:
        api_counts.plot(kind='barh', ax=ax8, color='steelblue')
        ax8.set_xlabel('Call Count')
        ax8.set_title('Top 10 Most Frequent API Calls')
        ax8.grid(True, alpha=0.3, axis='x')
    else:
        ax8.text(0.5, 0.5, 'No API calls recorded', ha='center', va='center',
                fontsize=12, transform=ax8.transAxes)
        ax8.set_title('Top 10 Most Frequent API Calls')
        ax8.axis('off')
    
    # Plot 9: Top 10 API Calls by Memory Usage
    ax9 = axes[2, 2]
    # Calculate total memory delta per API call type
    memory_by_api = (df.groupby('APICall')['Delta_VmRSS_KB'].sum() / 1024).sort_values(ascending=True).tail(10)
    if len(memory_by_api) > 0:
        memory_by_api.plot(kind='barh', ax=ax9, color='coral')
        ax9.set_xlabel('Total Memory Delta (MB)')
        ax9.set_title('Top 10 API Calls by Memory Impact')
        ax9.grid(True, alpha=0.3, axis='x')
    else:
        ax9.text(0.5, 0.5, 'No API calls recorded', ha='center', va='center',
                fontsize=12, transform=ax9.transAxes)
        ax9.set_title('Top 10 API Calls by Memory Impact')
        ax9.axis('off')
    
    # Plot 10: Memory Usage by API Call (average per call)
    ax10 = axes[3, 0]
    # Calculate average memory delta per API call type
    avg_memory_by_api = (df.groupby('APICall')['Delta_VmRSS_KB'].mean() / 1024).sort_values(ascending=True).tail(10)
    if len(avg_memory_by_api) > 0:
        avg_memory_by_api.plot(kind='barh', ax=ax10, color='mediumseagreen')
        ax10.set_xlabel('Avg Memory Delta per Call (MB)')
        ax10.set_title('Top 10 API Calls by Avg Memory per Call')
        ax10.grid(True, alpha=0.3, axis='x')
    else:
        ax10.text(0.5, 0.5, 'No API calls recorded', ha='center', va='center',
                fontsize=12, transform=ax10.transAxes)
        ax10.set_title('Top 10 API Calls by Avg Memory per Call')
        ax10.axis('off')
    
    # Plot 11: Cumulative memory by API over time
    ax11 = axes[3, 1]
    # Get top 5 API calls by total memory impact
    top5_apis = df.groupby('APICall')['Delta_VmRSS_KB'].sum().nlargest(5).index
    if len(top5_apis) > 0:
        for api in top5_apis:
            api_data = df[df['APICall'] == api]
            ax11.plot(api_data['TimeSec'], (api_data['Delta_VmRSS_KB'].cumsum() / 1024), label=api, linewidth=2)
        ax11.set_xlabel('Time (s)')
        ax11.set_ylabel('Cumulative Memory Delta (MB)')
        ax11.set_title('Cumulative Memory Impact by Top 5 APIs')
        ax11.legend(fontsize=8)
        ax11.grid(True, alpha=0.3)
    else:
        ax11.text(0.5, 0.5, 'No API calls recorded', ha='center', va='center',
                fontsize=12, transform=ax11.transAxes)
        ax11.set_title('Cumulative Memory Impact by Top 5 APIs')
        ax11.axis('off')
    
    # Plot 12: Leak detection timeline with annotations
    ax12 = axes[3, 2]
    if not leak_events.empty:
        # Show individual leak magnitudes over time
        ax12.bar(leak_events['TimeSec'], leak_events['PotentialLeak_KB'] / 1024, 
                width=0.01, color='red', alpha=0.7, label='Leak Magnitude')
        ax12.set_xlabel('Time (s)')
        ax12.set_ylabel('Leaked Memory (MB)')
        ax12.set_title('Individual Leak Events Timeline', fontweight='bold')
        ax12.legend()
        ax12.grid(True, alpha=0.3, axis='y')
        ax12.set_facecolor('#fff5f5')
        
        # Add text annotation for total
        total_leaked = leak_events['PotentialLeak_KB'].sum() / 1024
        ax12.text(0.95, 0.95, f'Total: {total_leaked:.2f} MB\n{len(leak_events)} events', 
                 transform=ax12.transAxes, ha='right', va='top',
                 bbox=dict(boxstyle='round', facecolor='white', alpha=0.8),
                 fontsize=10, fontweight='bold', color='darkred')
    else:
        ax12.text(0.5, 0.5, 'No Memory Leaks Detected!\n✓ All resources properly cleaned up', 
                 ha='center', va='center', fontsize=14, color='green', 
                 fontweight='bold', transform=ax12.transAxes)
        ax12.set_title('Individual Leak Events Timeline')
        ax12.axis('off')
    
    plt.tight_layout()
    
    # Save plot
    output_file = Path(csv_file).stem + '_plot.png'
    plt.savefig(output_file, dpi=150, bbox_inches='tight')
    print(f"Plot saved to: {output_file}")
    
    # Show plot
    plt.show()
    
    # Print summary statistics
    print("\n=== Summary Statistics ===")
    print(f"Total API calls tracked: {len(df)}")
    print(f"Time span: {df['TimeSec'].max():.2f} seconds ({df['TimeMs'].max():.2f} ms)")
    print(f"Peak VmRSS: {df['VmRSS_KB'].max():.2f} KB ({df['VmRSS_KB'].max()/1024:.2f} MB)")
    print(f"Peak VmSize: {df['VmSize_KB'].max():.2f} KB ({df['VmSize_KB'].max()/1024:.2f} MB)")
    print(f"Total memory allocated: {df['TotalMemory_Bytes'].max():.2f} bytes "
          f"({df['TotalMemory_Bytes'].max()/(1024*1024):.2f} MB)")
    print(f"Number of threads: {df['Threads'].max()}")
    
    # Print leak detection summary
    print(f"\n=== MEMORY LEAK DETECTION ===")
    if df['CumulativeLeak_KB'].max() > 0:
        print(f"⚠️  LEAKS DETECTED!")
        print(f"Total leaked memory: {df['CumulativeLeak_KB'].max() / 1024:.2f} MB ({df['CumulativeLeak_KB'].max():.2f} KB)")
        print(f"Number of leak events: {len(leak_events)}")
        if not leak_events.empty:
            print(f"\nLeak events by API:")
            leak_summary = leak_events.groupby('APICall')['PotentialLeak_KB'].agg(['count', 'sum', 'mean'])
            leak_summary.columns = ['Count', 'Total_KB', 'Avg_KB']
            leak_summary = leak_summary.sort_values('Total_KB', ascending=False)
            for api, row in leak_summary.iterrows():
                print(f"  {api}: {row['Count']} events, {row['Total_KB']/1024:.2f} MB total, {row['Avg_KB']/1024:.2f} MB avg")
    else:
        print(f"✓ No memory leaks detected!")
        print(f"  All resources were properly cleaned up.")
    
    print(f"\nPeak resource counts:")
    print(f"  Contexts: {df['Contexts'].max()}")
    print(f"  CommandQueues: {df['CommandQueues'].max()}")
    print(f"  Modules: {df['Modules'].max()}")
    print(f"  Kernels: {df['Kernels'].max()}")
    print(f"  CommandLists: {df['CommandLists'].max()}")
    print(f"  Events: {df['Events'].max()}")
    
    # Print top API calls by memory usage
    print(f"\n=== Top 10 API Calls by Total Memory Impact ===")
    memory_by_api = df.groupby('APICall')['Delta_VmRSS_KB'].sum().sort_values(ascending=False).head(10)
    for api, mem in memory_by_api.items():
        print(f"  {api}: {mem:.2f} KB ({mem/1024:.2f} MB)")
    
    print(f"\n=== Top 10 API Calls by Average Memory per Call ===")
    avg_memory_by_api = df.groupby('APICall')['Delta_VmRSS_KB'].mean().sort_values(ascending=False).head(10)
    for api, mem in avg_memory_by_api.items():
        count = len(df[df['APICall'] == api])
        print(f"  {api}: {mem:.2f} KB/call ({count} calls)")

if __name__ == '__main__':
    if len(sys.argv) != 2:
        print(__doc__)
        sys.exit(1)
    
    csv_file = sys.argv[1]
    if not Path(csv_file).exists():
        print(f"Error: File '{csv_file}' not found")
        sys.exit(1)
    
    try:
        import pandas
        import matplotlib
    except ImportError as e:
        print(f"Error: Required Python packages not installed")
        print(f"Install with: pip install pandas matplotlib")
        sys.exit(1)
    
    plot_resource_tracker(csv_file)