[kaggle] 캐글 필사 - Parkinson's Disease Progression Prediction 1

카테고리 없음

[kaggle] 캐글 필사 - Parkinson's Disease Progression Prediction 1

개발자열무 2023. 4. 27. 10:37

✋ Kaggle Competition - AMP®-Parkinson's Disease Progression Prediction

https://www.kaggle.com/competitions/amp-parkinsons-disease-progression-prediction

AMP®-Parkinson's Disease Progression Prediction | Kaggle

www.kaggle.com

✔ The Goal of Competition - 파킨슨병 환자의 진행을 측정하는 MDS-UPDR 점수를 예측하는 것

✔ 파킨슨병 환자와 연령이 일치하는 정상 대조군의 시간 경과에 따른

단백질 및 펩타이드 수치 데이터로 훈련된 모델을 개발

✋ Copying in kaggle New Notebook

✔ Copying Notebook : Simple Linear model with only clinical data

# write a read-only input directory for datasets
import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
        
/kaggle/input/amp-parkinsons-disease-progression-prediction/train_proteins.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/train_clinical_data.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/public_timeseries_testing_util.py
/kaggle/input/amp-parkinsons-disease-progression-prediction/supplemental_clinical_data.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/train_peptides.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/amp_pd_peptide/competition.cpython-37m-x86_64-linux-gnu.so
/kaggle/input/amp-parkinsons-disease-progression-prediction/amp_pd_peptide/__init__.py
/kaggle/input/amp-parkinsons-disease-progression-prediction/example_test_files/sample_submission.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/example_test_files/test_proteins.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/example_test_files/test_peptides.csv
/kaggle/input/amp-parkinsons-disease-progression-prediction/example_test_files/test.csv

✔ Loading Libraries

# Loading libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgb
import tqdm                      # 시간이 걸리는 작업의 경우 상태 확인을 위해 사용
import re                        # 정규표현 처리를 하기 위해 표준 라이브러리
from itertools import product    # 효율적인 루핑을 위한 위한 이터레이터를 만드는 함수
from functools import reduce     # 고차원 함수를 위해 고안

import warnings   # 경고제어
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=FutureWarning)

# 소수점 이하 3자리만 표현
pd.set_option('display.float_format',lambda x: '%.3f' % x)

✔ Reading the Datasets

# Reading the datasets
protein_data = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/train_proteins.csv')
peptides_data = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/train_peptides.csv')
target_data = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/train_clinical_data.csv')
sup_target_data = pd.read_csv('/kaggle/input/amp-parkinsons-disease-progression-prediction/supplemental_clinical_data.csv')
protein_data.shape, peptides_data.shape, target_data.shape

## ((232741, 5), (981834, 6), (2615, 8))

✔ target data + sup_target_data (Merging)

👉 모델링에 임상 데이터 (train_clinical_data = target_data)만 사용할 것이므로,

target_data 와 sup_target_data 병합하기

# Merging target data and sup_target_data since we will be using only clinical data for modelling
target_data = pd.concat([target_data,sup_target_data],axis = 0).reset_index(drop = True)  # 병합
target_data = target_data[target_data.visit_month != 5].copy()  # 의미없는 데이터 제거

# 유일값(unique) 확인
target_data.shape, target_data.visit_id.nunique(), target_data.patient_id.nunique(), target_data.visit_month.nunique()

## ((4720, 8), 4720, 1019, 17)

✔ target_data 결측치 채우기

👉 upd23b_clinical_state_on_medication 컬럼의 결측치를 unknown으로 채움

# 결측치 채우기
# upd23b_clinical_state_on_medication 안의 결측치를 unknown으로
target_data.upd23b_clinical_state_on_medication.fillna('unknown',inplace = True)

# target_data 결측치 확인
target_data.isna().sum()

# visit_id                                  0
# patient_id                                0
# visit_month                               0
# updrs_1                                  96
# updrs_2                                  98
# updrs_3                                  30
# updrs_4                                1863
# upd23b_clinical_state_on_medication       0
# dtype: int64

✔ 카테고리 컬럼 설정

# Category columns
id_cols = ['visit_id','patient_id','visit_month']
target_cols = ['updrs_1','updrs_2','updrs_3','updrs_4']
month_list  =  [0,6,12,24]

✔ create_target_features 함수 생성 (by target data)

def create_target_features(target_data,test=None):
    
    # test data가 없다면, target data의 'patient_id','visit_month','upd23b_clinical_state_on_medication'만
    # 포함된 dataframe 생성
    if test is None:
        test1 = target_data[['patient_id','visit_month','upd23b_clinical_state_on_medication']].copy()
        test1['upd23b_clinical_state_on_medication']  
        = test1['upd23b_clinical_state_on_medication'].fillna('unknown')
        
    # test data가 있다면, test data 복사하여 'upd23b_clinical_state_on_medication' 칼럼이 없다면
    # 'unknown' 값을 추가하고 결측값은 'unknown'으로 대체
    else:
        test1 = test.copy()
        if 'upd23b_clinical_state_on_medication' not in test1.columns.tolist():
            test1['upd23b_clinical_state_on_medication'] = 'unknown'
        else:
            test1['upd23b_clinical_state_on_medication']  
            = test1['upd23b_clinical_state_on_medication'].fillna('unknown')
            
        # 'patient_id', 'visit_month', 'upd23b_clinical_state_on_medication' 칼럼만 남기고 중복을 제거    
        test1 = test1[['patient_id','visit_month',
                       'upd23b_clinical_state_on_medication']].drop_duplicates().copy()
        
        # patient_id / visit_month 유일값 저장
        patients = test1.patient_id.unique()
        visit_months = test1.visit_month.unique()
        
        # test data 입력 시, 비교를 위해 original data 컬럼 설정
        test1['visit_month_orig'] = test1['visit_month']

        # test data가 제공되면 추가 데이터 처리를 수행하여
        # 각 환자의 모든 관련 방문 월을 포함하도록 데이터를 확장
        # 데이터의 각 환자를 반복하고 각 환자에 대해 각 방문 월(0, 6, 12, 24개월)에 대한
        # 환자 데이터의 사본을 생성 이후 데이터 복사본 연결하여 test1 data 생성
        test_data = pd.DataFrame()
        for patient in patients:
            for month in [0,6,12,24]:
                p_data = test1[test1['patient_id'] == patient].copy()
                p_data['visit_month'] = p_data['visit_month'].values + month
                test_data = pd.concat([test_data,p_data],axis=0)
        test1 = test_data.copy()
        
    return test1
    
    # visit_month & medication 결측치 대체
    target_data1 = target_data.copy()
    tmeds = target_data1.groupby(['visit_month','upd23b_clinical_state_on_medication']
                            ,group_keys =False)[target_cols].apply(lambda x: x.fillna(x.median())).sort_index()
    target_data1 = target_data1[['patient_id','visit_month','upd23b_clinical_state_on_medication']].join(tmeds)
    
    # grp features 생성
    # 각 visit_month와 medication 그룹에 대해 updrs_1 컬럼의 개수(count)를 생성
    all_grp_cols = [['visit_month'],['upd23b_clinical_state_on_medication']]
                    + [['visit_month','upd23b_clinical_state_on_medication']]
    
    target_data2 = target_data1[['visit_month','upd23b_clinical_state_on_medication']].drop_duplicates()
    for grp_col in all_grp_cols:    
        temp = target_data1.groupby(grp_col)[target_cols].agg(['min','max','mean','median','sum','std'])
        temp.columns = [i+'_' + j + '_'+ '_'.join(grp_col) for i,j in temp.columns]
        target_data2 = target_data2.join(temp,on = grp_col)
        target_data2  = target_data2.join(target_data1.groupby(grp_col)['updrs_1'].count().rename('_'.join(grp_col)+'_count')
                                         , on = grp_col)
                                         
    # 'med_unknown', 'med_off', 'med_on' 컬럼 추가
    # 'upd23b_clinical_state_on_medication' 값이 각각 'unknown', 'Off', 'On'인 경우 1, 그렇지 않은 경우 0
    test1['med_unknown'] = (test1['upd23b_clinical_state_on_medication'] == 'unknown').astype(int)
    test1['med_off'] = (test1['upd23b_clinical_state_on_medication'] == 'Off').astype(int)
    test1['med_on'] = (test1['upd23b_clinical_state_on_medication'] == 'On').astype(int)
    
    model_data = test1.merge(target_data2,on = ['visit_month','upd23b_clinical_state_on_medication']
                             , how = 'inner').drop(columns = 'upd23b_clinical_state_on_medication')
    feature_cols = model_data.drop(columns = ['patient_id']).columns.tolist()
    
    # model_data에 visit_month를 기준으로 중앙값으로 대체
    model_data = model_data.groupby('visit_month',group_keys = False).apply(lambda x: x.fillna(x.median()))
    
    # test data가 없다면 'patient_id', 'visit_month', target_cols 컬럼이 추가
    if test is None:
        model_data = model_data.merge(target_data1[['patient_id','visit_month'] + target_cols],
                                      on = ['patient_id','visit_month'],
                                      how = 'inner')

    return model_data,feature_cols

✔ feature 가져오기 (by create_target_features)

# Getting the features 
model_data, feature_cols = create_target_features(target_data)
model_data.shape, len(feature_cols)

## ((4720, 84), 79)


# 중복값 제거
model_data[['patient_id','visit_month']].drop_duplicates().shape, model_data.shape

## ((4720, 2), (4720, 84))

✔ feature_cols 설정

# feature_cols = ['visit_month','med_unknown','med_off','med_on','visit_month_count',
#                 'upd23b_clinical_state_on_medication_count','visit_month_upd23b_clinical_state_on_medication_count']
feature_cols = ['visit_month','visit_month_count']

저작자표시 변경금지 (새창열림)