Oneweb Antenna AIM Graphs

Created by Microcom Team, Modified on Wed, 27 Mar at 9:57 AM by Microcom Team

This article provides guidance on using a Python script to generate interactive graphs using Plotly for visualizing antenna path tracing data from AIM logs. The script decompresses the data files, processes them, and creates an HTML dashboard containing the interactive plots.

Script Overview:

The provided Python script is designed to generate interactive dashboards to visualize AIM data. Here's a brief overview of its functionality:

Decompression Functions: The script includes functions to decompress .tar.gz files containing data.
Data Processing: It extracts relevant data from the 'G_' files, including antenna path and SINR, and prepares it for visualization.
Dashboard Generation: Using Plotly, the script creates interactive polar plots for primary and secondary antenna data.
HTML Output: The generated plots are compiled into an HTML file for easy viewing and interaction.

Instructions for Usage:

Prerequisites:

Python3 installed on your system.
Necessary Python libraries (plotly, pandas, datetime, os, glob, re, tarfile, logging) installed. You can install missing libraries using pip(or pip3).
```
pip3 install plotly pandas re glob datetime os re tarfile logging
```

Steps to Use the Script:

Download the Script: Copy the provided script into a Python (.py) file on your system.
Prepare Data: Download the AIM logs or Complete AIM logs from the Oneweb LUI. By default the log.tar.gz is downloaded into the '~/Downloads' directory.
Run the Script:
- Open a terminal or command prompt.
- Navigate to the directory containing the script.
- Execute the script by running:
```
python3 aim_graphs.py
```
Input Tar File Name: When prompted, provide the name of the .tar.gz file containing the AIM data. Exclude the '.tar.gz' extension, e.g 'log'
Dashboard Generation:
- The script will process the data, create a folder under the input name, & create interactive polar plots, and compile them into an HTML file.
- Once completed, you will find the dashboard HTML file named dashboard.html in your '~/Desktop' directory.
View Dashboard:
- Navigate to your desktop and open the dashboard.html file in a web browser.
- Explore the interactive plots displaying antenna data.

Notes:

Ensure that your data files follow the expected naming conventions and directory structure for proper processing.
Check the logs generated by the script for any errors or warnings during execution.
Script compatibility verified on Linux & MacOS.

By following these instructions, you can effectively use the provided Python script to generate an interactive dashboard for visualizing antenna data. If you encounter any issues or have further questions, contact mgoodwin@microcom.tv.

import plotly.graph_objects as go
from plotly.subplots import make_subplots
import pandas as pd
from datetime import datetime
import os
import glob
import re
import tarfile
import logging

file_name = input("Tar file name( excluding '.tar.gz'):")
UTs = [file_name]
pref = "G_"
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)


def decompress_tar(file_path, output_dir):
    if os.path.exists(output_dir):
        logger.info(f'{output_dir} already exists. skipping extract.')
    else:
        with tarfile.open(file_path, 'r:gz') as tar:
            tar.extractall(output_dir)
            logger.info(f"{output_dir} Extraction complete.")


def decompress_subdir(file_path):
    master_tar = f"{file_path}/master.tar.gz"
    slave_tar = f"{file_path}/slave.tar.gz"
    master_path = f"{file_path}/master"
    slave_path = f"{file_path}/slave"

    decompress_tar(master_tar, master_path)
    decompress_tar(slave_tar, slave_path)


def decompress_bkup(file_path):
    master_backup_tar = f"{file_path}/master/backup.tar.gz"
    slave_backup_tar = f"{file_path}/slave/backup.tar.gz"
    master_backup_path = f"{file_path}/master/backup"
    slave_backup_path = f"{file_path}/slave/backup"

    decompress_tar(master_backup_tar, master_backup_path)
    decompress_tar(slave_backup_tar, slave_backup_path)


for log in UTs:
    try:
        pathing = os.path.join(os.path.expanduser('~'), f'Downloads/{log}')
        if not os.path.exists(pathing):
            decompress_tar(f'{pathing}.tar.gz', pathing)
            decompress_subdir(pathing)
            decompress_bkup(pathing)
            logging.info('Completed')
    except Exception as e:
        logger.error(f'Error processing {log}: {e}')


def extract_date(file_path):
    df = pd.read_csv(file_path, sep='\t', nrows=3, header=None)
    third_line = df.iloc[2, 0] if len(df) >= 3 else None
    date_match = re.search(r'Date (\d{4} \d{2} \d{2})', str(third_line))
    if date_match:
        extracted_date_str = date_match.group(1)
        date_object = datetime.strptime(extracted_date_str, "%Y %m %d")
        formatted_date = date_object.strftime("%d%b%y")
        logger.info(f'G file data formatting complete')
        return formatted_date
    else:
        logger.error(f'Error with {file_path} file Data')
        return None


def find_file(directory, prefix):
    pattern = os.path.join(directory, f'{prefix}*.txt')
    files = glob.glob(pattern)
    logging.info(f'Processing AIM log.')
    return files if files else None


def aim_dash(prefix):
    if prefix == 'P_':
        az = 'LAST_ACTUAL_AZ_P'
        el = 'LAST_ACTUAL_EL_P'
        sinr = 'LAST_AVG_SINR'
        azs = 'LAST_ACTUAL_AZ_S'
        els = 'LAST_ACTUAL_EL_S'
    elif prefix == "G_":
        az = azs = 'CURRENT_AZ'
        el = els = 'CURRENT_EL'
        sinr = 'SINR'

    log_tar = UTs
    all_figs = []

    def figures_to_html(figs, filename="dashboard.html"):
        with open(os.path.join(os.path.expanduser("~"), 'Desktop', filename), 'w') as dashboard:
            dashboard.write('''
            <html>
            <style>\n
            body { 
                font-family: HelveticaNeue;
                margin: 0;
                padding: 0;
                background-color: #111111;
                }\n
            #parent {
                margin:auto; 
                align-items:center; 
                justify-content:center; 
                height:100vh;
            </style>
            <head></head>
            <body><div id='parent'>''' + '\n')
            for fig in figs:
                inner_html = fig.to_html()
                dashboard.write(f"<div>{inner_html}</div>")
            dashboard.write("</div></body></html>" + "\n")
            logging.info(f'{filename} template created')

    for log in log_tar:
        path = path2 = os.path.join(os.path.expanduser('~'), 'Downloads', log, 'master', 'backup', '')
        path2 = os.path.join(os.path.expanduser('~'), 'Downloads', log, 'slave', 'backup', '') if prefix == 'G_' else path
        all_figs_for_log = []
        log_dates = []

        file_name = find_file(path, prefix)
        file_name2 = find_file(path2, prefix)

        # Manually create a list of subplots
        figs = [make_subplots(rows=1, cols=2, specs=[[{'type': 'polar'}] * 2] * 1,
                              subplot_titles=("Primary", "Secondary")) for _ in range(len(file_name))]

        for idx, (file, fig) in enumerate(zip(file_name, figs)):
            prim_df = pd.read_csv(file, sep=r'\t', engine='python', skiprows=3)
            prim_df[sinr] = prim_df[sinr].clip(lower=0)
            log_date = extract_date(file)
            log_dates.append(log_date)
            fig.add_trace(go.Scatterpolar(
                name='Primary',
                showlegend=False,
                theta=prim_df[az],
                r=prim_df[el], mode='markers',
                hovertext=prim_df["Time"],
                marker=dict(color=prim_df[sinr], colorscale='Viridis', showscale=True, opacity=0.75)),
                row=1,
                col=1,
            )

        for idx, (file, fig) in enumerate(zip(file_name2, figs)):
            sec_df = pd.read_csv(file, sep=r'\t', engine='python', skiprows=3)
            sec_df[sinr] = sec_df[sinr].clip(lower=0)

            fig.add_trace(go.Scatterpolar(
                name='Secondary',
                showlegend=False,
                theta=sec_df[azs],
                r=sec_df[els], mode='markers',
                hovertext=sec_df["Time"],
                marker=dict(color=sec_df[sinr], colorscale='Viridis', showscale=False, opacity=0.75)),
                row=1,
                col=2,
            )

        for log_date, fig in zip(log_dates, figs):  # Iterate over log_dates and figs together
            fig.update_layout(
                polar=dict(
                    radialaxis=dict(range=[90, 30], angle=22.5),
                    angularaxis=dict(rotation=90, direction='clockwise'),
                ),
                polar2=dict(
                    radialaxis=dict(range=[90, 30], angle=22.5),
                    angularaxis=dict(rotation=90, direction='clockwise'),
                ),
                template='plotly_dark',
                height=1000,
                title=f"{ log_date.upper()}: Antenna Forward SINR ",
                legend=dict(title_text="SINR", yanchor='top', y=1.0),
            )

            fig.update_traces(marker=dict(size=4 if prefix == "G_" else 3))
            all_figs_for_log.append(fig)

        all_figs.extend(all_figs_for_log)
        logging.info('All Figures Created')

    # Call figures_to_html once after accumulating all figures
    figures_to_html(all_figs)
    logging.info(f'Figures added to ~/Desktop/dashboard.html')


aim_dash(pref)