prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "fe7a8b04",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import mne\n",
    "import scipy.io, scipy.interpolate\n",
    "import plotly.express as px\n",
    "import pandas as pd\n",
    "\n",
    "# mne.utils.set_config('MNE_USE_CUDA', 'true')\n",
    "\n",
    "L_FREQ, H_FREQ = 40, 300 # Lower and upper filtration bounds\n",
    "CHANNELS_NUM = 62        # Number of channels in ECoG data\n",
    "WAVELET_NUM = 40         # Number of wavelets in the indicated frequency range, with which the convolution is performed\n",
    "DOWNSAMPLE_FS = 100      # Desired sampling rate\n",
    "time_delay_secs = 0.2    # Time delay hyperparameter\n",
    "\n",
    "current_fs = DOWNSAMPLE_FS\n",
    "\n",
    "def reshape_column_ecog_data(multichannel_signal: np.ndarray):\n",
    "    return multichannel_signal.T # (time, features) -> (features, time)\n",
    "\n",
    "def filter_ecog_data(multichannel_signal: np.ndarray, fs=1000, powerline_freq=50):\n",
    "    \"\"\"\n",
    "    Harmonics removal and frequency filtering\n",
    "    :param multichannel_signal: Initial multi-channel signal\n",
    "    :param fs: Sampling rate\n",
    "    :param powerline_freq: Grid frequency\n",
    "    :return: Filtered signal\n",
    "    \"\"\"\n",
    "    harmonics = np.array([i * powerline_freq for i in range(1, (fs // 2) // powerline_freq)])\n",
    "\n",
    "    print(\"Starting...\")\n",
    "    signal_filtered = mne.filter.filter_data(multichannel_signal,\n",
    "                                             fs, l_freq=L_FREQ, h_freq=H_FREQ)  # remove all frequencies between l and h\n",
    "    print(\"Noise frequencies removed...\")\n",
    "    signal_removed_powerline_noise = mne.filter.notch_filter(signal_filtered,\n",
    "                                                             fs, freqs=harmonics)  # remove powerline  noise\n",
    "    print(\"Powerline noise removed...\")\n",
    "    \n",
    "    return signal_removed_powerline_noise\n",
    "\n",
    "def normalize(multichannel_signal: np.ndarray, return_values = None):\n",
    "    \"\"\"\n",
    "    standardization and removal of the median  from each channel\n",
    "    :param multichannel_signal: Multi-channel signal\n",
    "    :param return_values: Whether to return standardization parameters. By default - no\n",
    "    \"\"\"\n",
    "    print(\"Normalizing...\")\n",
    "    means = np.mean(multichannel_signal, axis=1, keepdims=True)\n",
    "    stds = np.std(multichannel_signal, axis=1, keepdims=True)\n",
    "    transformed_data = (multichannel_signal - means) / stds\n",
    "    common_average = np.median(transformed_data, axis=0, keepdims=True)\n",
    "    transformed_data = transformed_data - common_average\n",
    "    if return_values:\n",
    "        return transformed_data, (means, stds)\n",
    "    print(\"Normalized...\")\n",
    "    return transformed_data\n",
    "\n",
    "def compute_spectrogramms(multichannel_signal : np.ndarray, fs=1000, freqs=np.logspace(np.log10(L_FREQ), np.log10(H_FREQ), WAVELET_NUM),\n",
    "                          output_type='power'):\n",
    "    \"\"\"\n",
    "    Compute spectrogramms using wavelet transforms\n",
    "\n",
    "    :param freqs: wavelet frequencies to uses\n",
    "    :param fs: Sampling rate\n",
    "    :return: Signal spectogramms in shape (channels, wavelets, time)\n",
    "    \"\"\"\n",
    "    \n",
    "    num_of_channels = len(multichannel_signal)\n",
    "\n",
    "    print(\"Computing wavelets...\")\n",
    "    spectrogramms = mne.time_frequency.tfr_array_morlet(multichannel_signal.reshape(1, num_of_channels, -1), sfreq=fs,\n",
    "                                                        freqs=freqs, output=output_type, verbose=10, n_jobs=6)[0]\n",
    "    \n",
    "    \n",
    "    print(\"Wavelet spectrogramm computed...\")\n",
    "    \n",
    "    return spectrogramms\n",
    "\n",
    "\n",
    "def downsample_spectrogramms(spectrogramms: np.ndarray, cur_fs=1000, needed_hz=H_FREQ, new_fs = None):\n",
    "    \"\"\"\n",
    "    Reducing the sampling rate of spectrograms\n",
    "    :param spectrogramms: Original set of spectrograms\n",
    "    :param cur_fs: Current sampling rate\n",
    "    :param needed_hz: The maximum frequency that must be unambiguously preserved during compression\n",
    "    :param new_fs: The required sampling rate (interchangeable with needed_hz)\n",
    "    :return: Decimated signal\n",
    "    \"\"\"\n",
    "    print(\"Downsampling spectrogramm...\")\n",
    "    if new_fs == None:\n",
    "        new_fs = needed_hz * 2    \n",
    "    downsampling_coef = cur_fs // new_fs\n",
    "    assert downsampling_coef > 1\n",
    "    downsampled_spectrogramm = spectrogramms[:, :, ::downsampling_coef]\n",
    "    print(\"Spectrogramm downsampled...\")\n",
    "    return downsampled_spectrogramm\n",
    "\n",
    "\n",
    "def normalize_spectrogramms_to_db(spectrogramms: np.ndarray, convert = False):\n",
    "    \"\"\"\n",
    "    Optional conversion to db, not used in the final version\n",
    "    \"\"\"\n",
    "    if convert:\n",
    "        return np.log10(spectrogramms+1e-12)\n",
    "    else:\n",
    "        return spectrogramms\n",
    "\n",
    "\n",
    "def interpolate_fingerflex(finger_flex, cur_fs=1000, true_fs=25, needed_hz=DOWNSAMPLE_FS, interp_type='cubic'):\n",
    "    \n",
    "    \"\"\"\n",
    "    Interpolation of the finger motion recording to match the new sampling rate\n",
    "    :param finger_flex: Initial sequences with finger flexions data\n",
    "    :param cur_fs: ECoG sampling rate\n",
    "    :param true_fs: Actual finger motions recording sampling rate\n",
    "    :param needed_hz: Required sampling rate\n",
    "    :param interp_type: Type of interpolation. By default - cubic\n",
    "    :return: Returns an interpolated set of finger motions with the desired sampling rate\n",
    "    \"\"\"\n",
    "    \n",
    "    print(\"Interpolating fingerflex...\")\n",
    "    downscaling_ratio = cur_fs // true_fs\n",
    "    print(\"Computing true_fs values...\")\n",
    "    finger_flex_true_fs = finger_flex[:, ::downscaling_ratio]\n",
    "    finger_flex_true_fs = np.c_[finger_flex_true_fs,\n",
    "        finger_flex_true_fs.T[-1]]  # Add as the last value on the interpolation edge the last recorded\n",
    "    # Because otherwise it is not clear how to interpolate the tail at the end\n",
    "\n",
    "    upscaling_ratio = needed_hz // true_fs\n",
    "    \n",
    "    ts = np.asarray(range(finger_flex_true_fs.shape[1])) * upscaling_ratio\n",
    "    \n",
    "    print(\"Making funcs...\")\n",
    "    interpolated_finger_flex_funcs = [scipy.interpolate.interp1d(ts, finger_flex_true_fs_ch, kind=interp_type) for\n",
    "                                     finger_flex_true_fs_ch in finger_flex_true_fs]\n",
    "    ts_needed_hz = np.asarray(range(finger_flex_true_fs.shape[1] * upscaling_ratio)[\n",
    "                              :-upscaling_ratio])  # Removing the extra added edge\n",
    "    \n",
    "    print(\"Interpolating with needed frequency\")\n",
    "    interpolated_finger_flex = np.array([[interpolated_finger_flex_func(t) for t in ts_needed_hz] for\n",
    "                                         interpolated_finger_flex_func in interpolated_finger_flex_funcs])\n",
    "    return interpolated_finger_flex\n",
    "\n",
    "\n",
    "def crop_for_time_delay(finger_flex : np.ndarray, spectrogramms : np.ndarray, time_delay_sec : float, fs : int):\n",
    "    \"\"\"\n",
    "    Taking into account the delay between brain waves and movements\n",
    "    :param finger_flex: Finger flexions\n",
    "    :param spectrogramms: Computed spectrogramms\n",
    "    :param time_delay_sec: time delay hyperparameter\n",
    "    :param fs: Sampling rate\n",
    "    :return: Shifted series with a delay\n",
    "    \"\"\"\n",
    "\n",
    "    time_delay = int(time_delay_sec*fs)\n",
    "\n",
    "    # the first motions do not depend on available data\n",
    "    finger_flex_cropped = finger_flex[..., time_delay:] \n",
    "    # The latter spectrograms have no corresponding data\n",
    "    spectrogramms_cropped = spectrogramms[..., :spectrogramms.shape[2]-time_delay]\n",
    "    return finger_flex_cropped, spectrogramms_cropped\n",
    "\n",
    "\n",
    "def visualize_signal(multichannel_signal: np.ndarray, channel_num: int, second_num: int, fs=DOWNSAMPLE_FS):\n",
    "    \"\"\"\n",
    "    Function to visualize multi-channel signal section\n",
    "    :param multichannel_signal: Multi-channel signal\n",
    "    :param channel_num: Channel selected for visualization\n",
    "    :param second_num: Selected record second\n",
    "    :param fs: Sampling rate\n",
    "    :return: -\n",
    "    \"\"\"\n",
    "    df_channel = pd.DataFrame(data=np.asarray([np.asarray(range(fs)),\n",
    "                                               multichannel_signal[channel_num][second_num*fs:second_num*fs+fs]]).T,\n",
    "                              index=range(fs), columns=[\"t\", \"V\"])\n",
    "\n",
    "    fig = px.line(df_channel, x=\"t\", y=\"V\", title=f'channel_{channel_num}')\n",
    "    fig.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b5fb018a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Interpolating fingerflex...\n",
      "Computing true_fs values...\n",
      "Making funcs...\n",
      "Interpolating with needed frequency\n",
      "Normalizing...\n",
      "Normalized...\n",
      "Starting...\n",
      "Setting up band-pass filter from 40 - 3e+02 Hz\n",
      "\n",
      "FIR filter parameters\n",
      "---------------------\n",
      "Designing a one-pass, zero-phase, non-causal bandpass filter:\n",
      "- Windowed time-domain design (firwin) method\n",
      "- Hamming window with 0.0194 passband ripple and 53 dB stopband attenuation\n",
      "- Lower passband edge: 40.00\n",
      "- Lower transition bandwidth: 10.00 Hz (-6 dB cutoff frequency: 35.00 Hz)\n",
      "- Upper passband edge: 300.00 Hz\n",
      "- Upper transition bandwidth: 75.00 Hz (-6 dB cutoff frequency: 337.50 Hz)\n",
      "- Filter length: 331 samples (0.331 sec)\n",
      "\n",
      "Noise frequencies removed...\n",
      "Setting up band-stop filter\n",
      "\n",
      "FIR filter parameters\n",
      "---------------------\n",
      "Designing a one-pass, zero-phase, non-causal bandstop filter:\n",
      "- Windowed time-domain design (firwin) method\n",
      "- Hamming window with 0.0194 passband ripple and 53 dB stopband attenuation\n",
      "- Lower transition bandwidth: 0.50 Hz\n",
      "- Upper transition bandwidth: 0.50 Hz\n",
      "- Filter length: 6601 samples (6.601 sec)\n",
      "\n",
      "Powerline noise removed...\n",
      "Computing wavelets...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done  12 tasks      | elapsed:   10.0s\n",
      "[Parallel(n_jobs=6)]: Done  62 out of  62 | elapsed:   36.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wavelet spectrogramm computed...\n",
      "Downsampling spectrogramm...\n",
      "Spectrogramm downsampled...\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Loading the raw training data and applying the processing algorithm\n",
    "\"\"\"\n",
    "\n",
    "import pathlib\n",
    "\n",
    "PATH = f\"{pathlib.Path().resolve()}/data/pure_data/\"\n",
    "\n",
    "data = scipy.io.loadmat(f'{PATH}/sub1_comp.mat')\n",
    "\n",
    "interpolated_finger_flex = interpolate_fingerflex(finger_flex=\n",
    "                           reshape_column_ecog_data(data['train_dg'].astype('float64')))\n",
    "\n",
    "db_spectrogramms = normalize_spectrogramms_to_db(spectrogramms=\n",
    "                   downsample_spectrogramms(spectrogramms=\n",
    "                   compute_spectrogramms(multichannel_signal=\n",
    "                   filter_ecog_data(multichannel_signal=\n",
    "                   normalize(multichannel_signal=\n",
    "                   reshape_column_ecog_data(data['train_data'].astype('float64'))))), new_fs = DOWNSAMPLE_FS))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1385297a",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Interpolating fingerflex...\n",
      "Computing true_fs values...\n",
      "Making funcs...\n",
      "Interpolating with needed frequency\n",
      "Normalizing...\n",
      "Normalized...\n",
      "Starting...\n",
      "Setting up band-pass filter from 40 - 3e+02 Hz\n",
      "\n",
      "FIR filter parameters\n",
      "---------------------\n",
      "Designing a one-pass, zero-phase, non-causal bandpass filter:\n",
      "- Windowed time-domain design (firwin) method\n",
      "- Hamming window with 0.0194 passband ripple and 53 dB stopband attenuation\n",
      "- Lower passband edge: 40.00\n",
      "- Lower transition bandwidth: 10.00 Hz (-6 dB cutoff frequency: 35.00 Hz)\n",
      "- Upper passband edge: 300.00 Hz\n",
      "- Upper transition bandwidth: 75.00 Hz (-6 dB cutoff frequency: 337.50 Hz)\n",
      "- Filter length: 331 samples (0.331 sec)\n",
      "\n",
      "Noise frequencies removed...\n",
      "Setting up band-stop filter\n",
      "\n",
      "FIR filter parameters\n",
      "---------------------\n",
      "Designing a one-pass, zero-phase, non-causal bandstop filter:\n",
      "- Windowed time-domain design (firwin) method\n",
      "- Hamming window with 0.0194 passband ripple and 53 dB stopband attenuation\n",
      "- Lower transition bandwidth: 0.50 Hz\n",
      "- Upper transition bandwidth: 0.50 Hz\n",
      "- Filter length: 6601 samples (6.601 sec)\n",
      "\n",
      "Powerline noise removed...\n",
      "Computing wavelets...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=6)]: Using backend LokyBackend with 6 concurrent workers.\n",
      "[Parallel(n_jobs=6)]: Done  12 tasks      | elapsed:    4.1s\n",
      "[Parallel(n_jobs=6)]: Done  62 out of  62 | elapsed:   17.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wavelet spectrogramm computed...\n",
      "Downsampling spectrogramm...\n",
      "Spectrogramm downsampled...\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Loading the raw validation data and applying the processing algorithm\n",
    "\"\"\"\n",
    "\n",
    "data_2 = scipy.io.loadmat(f'{PATH}/sub1_testlabels.mat')\n",
    "\n",
    "interpolated_finger_flex_val = interpolate_fingerflex(finger_flex=\n",
    "                                                      reshape_column_ecog_data(data_2['test_dg'].astype('float64')))\n",
    "\n",
    "\n",
    "db_spectrogramms_val = normalize_spectrogramms_to_db(spectrogramms=\n",
    "                       downsample_spectrogramms(spectrogramms=\n",
    "                       compute_spectrogramms(multichannel_signal=\n",
    "                       filter_ecog_data(multichannel_signal=\n",
    "                       normalize(multichannel_signal=\n",
    "                       reshape_column_ecog_data(data['test_data'].astype('float64'))))), new_fs=DOWNSAMPLE_FS))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "6f405cee",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 39980)\n",
      "(62, 40, 39980)\n",
      "(5, 19980)\n",
      "(62, 40, 19980)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Taking time delay into account\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "interpolated_finger_flex_cropped, db_spectrogramms_cropped = crop_for_time_delay(interpolated_finger_flex,\n",
    "                                                                                db_spectrogramms, time_delay_secs,\n",
    "                                                                                current_fs)\n",
    "interpolated_finger_flex_val_cropped, db_spectrogramms_val_cropped = crop_for_time_delay(interpolated_finger_flex_val,\n",
    "                                                                                db_spectrogramms_val, time_delay_secs,\n",
    "                                                                                current_fs)\n",
    "\n",
    "print(interpolated_finger_flex_cropped.shape)\n",
    "print(db_spectrogramms_cropped.shape)\n",
    "print(interpolated_finger_flex_val_cropped.shape)\n",
    "print(db_spectrogramms_val_cropped.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0fe4373e",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Saving processed data\n",
    "\"\"\"\n",
    "\n",
    "import pathlib,os\n",
    "SAVE_PATH = \"/home/lomtev/brain_train/data/\"\n",
    "def save_proccessed_data(ecog_data, fingerflex_data, path, val=None, add_name = \"\", reshape = False):\n",
    "    pathlib.Path(f\"{path}/train\").mkdir(parents=True, exist_ok=True)\n",
    "    pathlib.Path(f\"{path}/val\").mkdir(parents=True, exist_ok=True)\n",
    "    pathlib.Path(f\"{path}/test\").mkdir(parents=True, exist_ok=True)\n",
    "    ecog_path = f\"{path}/train/ecog_data{add_name}.npy\" if val is None else f\"{path}/val/ecog_data{add_name}.npy\" if \\\n",
    "    val is True else f\"{path}/test/ecog_data{add_name}.npy\"\n",
    "    fingerflex_path = f\"{path}/train/fingerflex_data{add_name}.npy\" if val is None else f\"{path}/val/fingerflex_data{add_name}.npy\" if \\\n",
    "        val is True else f\"{path}/test/fingerflex_data{add_name}.npy\"\n",
    "    \n",
    "    if reshape:\n",
    "        ecog_data = ecog_data.reshape(CHANNELS_NUM*WAVELET_NUM,-1)\n",
    "    \n",
    "    os.remove(ecog_path)\n",
    "    os.remove(fingerflex_path)\n",
    "    np.save(ecog_path, ecog_data)\n",
    "    np.save(fingerflex_path, fingerflex_data)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d97d69f0",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "save_proccessed_data(db_spectrogramms_cropped, interpolated_finger_flex_cropped, SAVE_PATH, add_name = \"\")\n",
    "save_proccessed_data(db_spectrogramms_val_cropped, interpolated_finger_flex_val_cropped, SAVE_PATH, val=True, add_name = \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ed76a888",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Loading processed data\n",
    "\"\"\"\n",
    "\n",
    "def load_data(ecog_data_path, fingerflex_data_path):\n",
    "    ecog_data = np.load(ecog_data_path)\n",
    "    fingerflex_data = np.load(fingerflex_data_path)\n",
    "    return ecog_data, fingerflex_data\n",
    "\n",
    "ecog_data, fingerflex_data = load_data(f\"{SAVE_PATH}/train/ecog_data.npy\", f\"{SAVE_PATH}/train/fingerflex_data.npy\")\n",
    "\n",
    "ecog_data_val, fingerflex_data_val = load_data(f\"{SAVE_PATH}/val/ecog_data.npy\", f\"{SAVE_PATH}/val/fingerflex_data.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e6460287",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 39980)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fingerflex_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4775209d",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5, 39980) (5, 19980)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Finger motions scaling\n",
    "\"\"\"\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "scaler.fit(fingerflex_data.T)\n",
    "\n",
    "fingerflex_data_scaled = scaler.transform(fingerflex_data.T).T\n",
    "fingerflex_data_val_scaled = scaler.transform(fingerflex_data_val.T).T\n",
    "\n",
    "print(fingerflex_data_scaled.shape, fingerflex_data_val_scaled.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "baf939b1",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "save_proccessed_data(ecog_data, fingerflex_data_scaled, SAVE_PATH, add_name = \"\")\n",
    "save_proccessed_data(ecog_data_val, fingerflex_data_val_scaled, SAVE_PATH, val=True, add_name = \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "bf1eef11",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62, 40, 39980)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ecog_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c96d1788",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(62, 40, 39980) (62, 40, 19980)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "ECoG data scaling\n",
    "\"\"\"\n",
    "\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "\n",
    "transformer = RobustScaler(unit_variance=True, quantile_range=(0.1, 0.9))\n",
    "transformer.fit(ecog_data.T.reshape(-1,WAVELET_NUM*CHANNELS_NUM))\n",
    "\n",
    "ecog_data_scaled = transformer.transform(ecog_data.T.reshape(-1,WAVELET_NUM*CHANNELS_NUM)).reshape(-1,\\\n",
    "                                                                                WAVELET_NUM, CHANNELS_NUM).T\n",
    "\n",
    "ecog_data_val_scaled = transformer.transform(ecog_data_val.T.reshape(-1,WAVELET_NUM*CHANNELS_NUM)).reshape(-1,\\\n",
    "                                                                                WAVELET_NUM, CHANNELS_NUM).T\n",
    "\n",
    "print(ecog_data_scaled.shape, ecog_data_val_scaled.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "3f087866",
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "save_proccessed_data(ecog_data_scaled, fingerflex_data_scaled, SAVE_PATH, add_name = \"\")\n",
    "save_proccessed_data(ecog_data_val_scaled, fingerflex_data_val_scaled, SAVE_PATH, val=True, add_name = \"\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml_env",
   "language": "python",
   "name": "ml_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}