MatfiletoNetCDF_CL_ARTofMELT_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This python script converts the processed .mat file for native Ceilometer observations into a user-friendly NetCDF file. It also converts all time averaged .mat files into their respecitve netCDF files. These final (four) netCDF files are found in the Bolin Centre Database. Data from the ARTofMELT expedition in Spring 2023. This code is written by Sonja Murto (2024)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import modules\n",
    "import sys,glob,os\n",
    "import numpy as np\n",
    "import math\n",
    "import pandas as pd\n",
    "import matplotlib.dates as mdates\n",
    "import xarray as xr\n",
    "import time,datetime\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import string\n",
    "import scipy.io\n",
    "import matplotlib.ticker as mticker\n",
    "from collections import Counter\n",
    "\n",
    "\n",
    "#metpy functions for thermodynamical variable conversions\n",
    "import metpy.calc as mpcalc\n",
    "from metpy.units import units\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define functions\n",
    "def datestring_fromtuple(date=(2001,2,15,12)):\n",
    "    '''\n",
    "    Input: a date tuple (YYYY,M,D,h)\n",
    "    Output: Date string: 'YYYYMMDD_hh'\n",
    "    '''\n",
    "    strdate=str(date[0])+\"{:0>2}\".format(str(date[1]))+\"{:0>2}\".format(str(date[2]))+'_'+\"{:0>2}\".format(str(date[3]))\n",
    "    return strdate\n",
    "\n",
    "def datetuple(date):\n",
    "    '''date is as datetime object, i.e. from pandas using .to_pydatetime()\n",
    "    '''\n",
    "    return date.year,date.month,date.day,date.hour\n",
    "\n",
    "def strtodatetuple(datestr='19900109_12',format_st='%Y%m%d_%H',returntuple=True):\n",
    "    if returntuple:   \n",
    "        return datetuple(datetime.datetime.strptime(datestr, format_st))\n",
    "    else:\n",
    "        return datetime.datetime.strptime(datestr, format_st)\n",
    "\n",
    "\n",
    "def returndatetime_fromdoy(doys, year=2023):\n",
    "    '''\n",
    "    fucntion to return datetime from DOY dates. \n",
    "    Takes the leap year into account if given for a leap year\n",
    "    Input: list of DOYs\n",
    "    Output: List of Dates (in datetime) corresponding to the given doys\n",
    "    Note, returns in microseconds.\n",
    "    If second rounded: [pd.to_datetime(T).round('1s') for T in date]\n",
    "    \n",
    "    '''\n",
    "    jdate_frac=doys\n",
    "    date=[]\n",
    "    for d in jdate_frac:\n",
    "        year,julian = [year,d]\n",
    "        date.append(datetime.datetime(year, 1, 1)+datetime.timedelta(days=julian -1))\n",
    "    return date\n",
    "\n",
    "def getList(dict): \n",
    "    '''\n",
    "    This function returns a list of keys for a pandas dictionary\n",
    "    '''\n",
    "    list = [] \n",
    "    for key in dict.keys(): \n",
    "        list.append(key) \n",
    "    return list\n",
    "\n",
    "def truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100,returncolors=False):\n",
    "    '''\n",
    "    This function creates a new colormap from the cmap chosen, taking the limits from minval and maxval, and the number of colors as n.\n",
    "    The default is a colormap as LinearSegmentedColormap, which can be used as colormap for imshow, contourf...\n",
    "    If you assign returncolors as True, the there will be an array of n amount of colors from the chosen colormap. This can be used in plots when each event has a color (n=50)\n",
    "    '''\n",
    "    new_cmap = mcolors.LinearSegmentedColormap.from_list('new_cmap', plt.get_cmap(cmap)(np.linspace(minval, maxval, n)))\n",
    "    if returncolors:\n",
    "        new_cmap=plt.get_cmap(cmap)(np.linspace(minval, maxval, n))\n",
    "    return new_cmap   \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define directories - change according to your computer settings\n",
    "cwd = os.getcwd() \n",
    "savefigpath = cwd + '/FIGS/Example_figs/' #directory for figures\n",
    "#directory where to find the data (.mat files)\n",
    "load_data = r'/Volumes/My Passport for Mac/MISU_sticka/WORK/viktigapapper/ARTofMELT_2023/AoM_data_mod/WX/cl31_output/'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert ceilometer data from .mat files into .nc files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create secondary dimensions\n",
    "cloud_layer_levs=np.arange(1,4,1).astype('int32') # 3 levels\n",
    "sc_layer_levs=np.arange(1,6,1).astype('int32') # 5 levels\n",
    "range_levs=np.arange(1,771,1).astype('int32') # 770 levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3] [1 2 3 4 5]\n"
     ]
    }
   ],
   "source": [
    "print(cloud_layer_levs,sc_layer_levs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 30s data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load 30s native data\n",
    "CL_SM_org = scipy.io.loadmat(load_data + 'CL31_ceilometer_ARTofMELT_20230507_20230613_30s_v01.mat',struct_as_record=True) #30s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['__header__', '__version__', '__globals__', 'cl31'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_SM_org.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('mday', 'doy', 'cloudcode', 'base_ht', 'vert_vis', 'high_sig', 'sc_frac', 'sc_ht', 'bs_prof', 'ceil_range', 'time')\n"