Environment Classification

Solution for submission 154921

A detailed solution for submission 154921 submitted for challenge Environment Classification

BanKhv
In [1]:
%matplotlib inline

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from tqdm import tqdm
import copy
import gc
import os
from glob import glob
import pickle
import random
import shutil
import seaborn as sns
from collections import Counter

from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold, cross_val_score, train_test_split
from lightgbm import LGBMClassifier, LGBMRegressor
from sklearn.metrics import roc_auc_score, f1_score, log_loss, accuracy_score, matthews_corrcoef
from sklearn.metrics import mean_squared_error

from sklearn.decomposition import TruncatedSVD
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans

#import category_encoders as ce
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, LabelEncoder

from skopt.space import Real, Categorical, Integer
from skopt.utils import use_named_args
from skopt import gp_minimize
from sklearn.metrics import classification_report
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

import warnings
warnings.simplefilter(action              = 'ignore', category = FutureWarning)
warnings.simplefilter(action              = 'ignore', category = DeprecationWarning)
warnings.simplefilter(action              = 'ignore', category = UserWarning)
warnings.simplefilter(action              = 'ignore', category = RuntimeWarning)
warnings.filterwarnings("ignore", message = "numpy.dtype size changed")
warnings.filterwarnings("ignore", message = "numpy.ufunc size changed")
pd.options.mode.chained_assignment = None
!pip install natsort!conda install -c conda-forge opencv --y%load_ext aicrowd.magic %aicrowd login# Downloading the Dataset !rm -rf data !mkdir data %aicrowd ds dl -c environment-classification -o data# Unzipping and Organising the datasets !unzip data/images.zip -d data/images > /dev/null
In [ ]:

In [2]:
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
import seaborn as sns
from glob import glob
from natsort import natsorted
import cv2
from PIL import Image

images_folder = "./data/images"
images_list = natsorted(glob(images_folder + "/*"))
img = Image.open(random.choice(images_list))
#img
In [3]:
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.models import Sequential

resnet = ResNet50(include_top = False, pooling = 'avg', weights = 'imagenet')
rmodel = Sequential()
rmodel.add(resnet)
rmodel.layers[0].trainable = False
In [ ]:

In [4]:
%%time
from tensorflow.keras.applications.resnet50 import preprocess_input
import cv2 
import numpy as np

resnet_feature_list = []
for file in tqdm(images_list):
    #print(file)
    im = cv2.imread(file)
    #im = cv2.resize(im,(256,256))
    img = preprocess_input(np.expand_dims(im.copy(), axis = 0))
    resnet_feature    = rmodel.predict(img)
    resnet_feature_np = np.array(resnet_feature)
    resnet_feature_list.append(resnet_feature_np.flatten())

array = np.array(resnet_feature_list)
100%|██████████| 700/700 [00:44<00:00, 15.68it/s]
CPU times: user 44.6 s, sys: 1.42 s, total: 46 s
Wall time: 44.7 s

In [5]:
pd.DataFrame(array).head()
Out[5]:
0 1 2 3 4 5 6 7 8 9 ... 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047
0 0.238505 0.181578 0.243401 0.209939 0.099278 0.118917 1.791079 0.078583 1.657051 0.435764 ... 0.020885 0.033485 0.114613 0.018791 0.192425 0.024448 0.101037 1.187931 0.076827 0.200750
1 0.042626 0.261808 0.092415 0.138349 0.148525 0.051944 1.048423 0.021261 0.805323 0.566928 ... 0.008454 0.098952 0.370662 0.045579 0.315647 0.109624 1.054739 0.202752 0.004386 0.221591
2 0.042169 0.144187 0.023210 0.291304 0.043215 0.052441 1.991662 0.248343 1.483952 0.520328 ... 0.036159 0.067551 0.322716 0.000239 0.274926 0.082152 0.320264 0.696963 0.013563 0.296884
3 0.572864 0.028532 0.075293 0.030276 0.045119 0.005722 1.342387 0.180273 0.633458 0.166158 ... 0.001467 0.038358 0.077849 0.000000 0.072513 0.081316 0.432226 0.418233 0.009666 0.436215
4 0.118663 0.095322 0.118683 0.087925 0.087147 0.021571 2.140892 0.432957 0.779140 0.195660 ... 0.046421 0.019373 0.440849 0.019678 0.050837 0.015758 1.513407 0.826788 0.016967 0.164996

5 rows × 2048 columns

In [6]:
model = KMeans(n_clusters = 5, random_state = 3)

predictions = model.fit_predict(array)

predictions.shape
Out[6]:
(700,)
1 - 0.587 2 - 0.513import random predictions = 0 seeds = 5 for i in range(seeds): xdf = df.copy() #r = random.choice(xdf.index) k = random.choice(xdf.columns) #xdf.drop(r, inplace = True) del xdf[k] print(xdf.shape) model.fit(xdf.values) pred = model.predict(xdf) print(Counter(pred)) predictions += pred predictions = predictions / seeds predictions = np.round(predictions)
In [7]:
df = pd.DataFrame({"ImageID":range(700), "label":predictions})
df['ImageID'] = df["ImageID"].astype(int)
df['label'] = df["label"].astype(int)
df = df.sort_values("ImageID").reset_index(drop = True)
df
Out[7]:
ImageID label
0 0 0
1 1 2
2 2 2
3 3 1
4 4 0
... ... ...
695 695 1
696 696 1
697 697 0
698 698 2
699 699 1

700 rows × 2 columns

In [8]:
!rm -rf assets
!mkdir assets

df.to_csv(os.path.join("assets", "submission.csv"), index = False)
In [ ]:

In [9]:
Counter(df.label)
Out[9]:
Counter({0: 113, 2: 231, 1: 226, 4: 109, 3: 21})
0.587 Counter({0: 131, 1: 198, 3: 213, 4: 106, 2: 52}) 0.513 Counter({0: 113, 2: 231, 1: 226, 4: 109, 3: 21})
In [ ]:

In [10]:
%load_ext aicrowd.magic
%aicrowd login
Please login here: https://api.aicrowd.com/auth/nw7lelMUJVKjPBU2jSdZOuPtlwU1tfE3r8pFZceHwjA
API Key valid
Saved API Key successfully!
Using notebook: /home/vlad/Games/AI6/Environment Classification/01/09.ipynb for submission...
Removing existing files from submission directory...
Scrubbing API keys from the notebook...
Collecting notebook...
submission.zip ━━━━━━━━━━━━━━━━━━━━━ 100.0%11.1/9.5 KB500.2 kB/s0:00:00B • ?-:--:--
                                                       ╭─────────────────────────╮                                                       
                                                       │ Successfully submitted! │                                                       
                                                       ╰─────────────────────────╯                                                       
                                                             Important links                                                             
┌──────────────────┬────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│  This submission │ https://www.aicrowd.com/challenges/ai-blitz-xi/problems/environment-classification/submissions/154918              │
│                  │                                                                                                                    │
│  All submissions │ https://www.aicrowd.com/challenges/ai-blitz-xi/problems/environment-classification/submissions?my_submissions=true │
│                  │                                                                                                                    │
│      Leaderboard │ https://www.aicrowd.com/challenges/ai-blitz-xi/problems/environment-classification/leaderboards                    │
│                  │                                                                                                                    │
│ Discussion forum │ https://discourse.aicrowd.com/c/ai-blitz-xi                                                                        │
│                  │                                                                                                                    │
│   Challenge page │ https://www.aicrowd.com/challenges/ai-blitz-xi/problems/environment-classification                                 │
└──────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
In [ ]:


Comments

You must login before you can post a comment.

Execute