曾桉提交
In [22]:
!pip install aicrowd-cli pyarrow
In [23]:
!aicrowd login
In [24]:
!aicrowd dataset download --challenge task-1-next-product-recommendation
In [25]:
import os
import numpy as np
import pandas as pd
from functools import lru_cache
In [26]:
train_data_dir = '.'
test_data_dir = '.'
task = 'task1'
PREDS_PER_SESSION = 100
In [27]:
# Cache loading of data for multiple calls
@lru_cache(maxsize=1)
def read_product_data():
return pd.read_csv(os.path.join(train_data_dir, 'products_train.csv'))
@lru_cache(maxsize=1)
def read_train_data():
return pd.read_csv(os.path.join(train_data_dir, 'sessions_train.csv'))
@lru_cache(maxsize=3)
def read_test_data(task):
return pd.read_csv(os.path.join(test_data_dir, f'sessions_test_{task}.csv'))
In [28]:
def read_locale_data(locale, task):
products = read_product_data().query(f'locale == "{locale}"')
sess_train = read_train_data().query(f'locale == "{locale}"')
sess_test = read_test_data(task).query(f'locale == "{locale}"')
return products, sess_train, sess_test
def show_locale_info(locale, task):
products, sess_train, sess_test = read_locale_data(locale, task)
train_l = sess_train['prev_items'].apply(lambda sess: len(sess))
test_l = sess_test['prev_items'].apply(lambda sess: len(sess))
print(f"Locale: {locale} \n"
f"Number of products: {products['id'].nunique()} \n"
f"Number of train sessions: {len(sess_train)} \n"
f"Train session lengths - "
f"Mean: {train_l.mean():.2f} | Median {train_l.median():.2f} | "
f"Min: {train_l.min():.2f} | Max {train_l.max():.2f} \n"
f"Number of test sessions: {len(sess_test)}"
)
if len(sess_test) > 0:
print(
f"Test session lengths - "
f"Mean: {test_l.mean():.2f} | Median {test_l.median():.2f} | "
f"Min: {test_l.min():.2f} | Max {test_l.max():.2f} \n"
)
print("======================================================================== \n")
In [29]:
products = read_product_data()
locale_names = products['locale'].unique()
for locale in locale_names:
show_locale_info(locale, task)
In [30]:
products.sample(5)
Out[30]:
In [31]:
train_sessions = read_train_data()
train_sessions.sample(5)
Out[31]:
In [32]:
test_sessions = read_test_data(task)
test_sessions.sample(5)
Out[32]:
In [33]:
def random_predicitons(locale, sess_test_locale):
random_state = np.random.RandomState(42)
products = read_product_data().query(f'locale == "{locale}"')
predictions = []
for _ in range(len(sess_test_locale)):
predictions.append(
list(products['id'].sample(PREDS_PER_SESSION, replace=True, random_state=random_state))
)
sess_test_locale['next_item_prediction'] = predictions
sess_test_locale.drop('prev_items', inplace=True, axis=1)
return sess_test_locale
In [34]:
test_sessions = read_test_data(task)
predictions = []
test_locale_names = test_sessions['locale'].unique()
for locale in test_locale_names:
sess_test_locale = test_sessions.query(f'locale == "{locale}"').copy()
predictions.append(
random_predicitons(locale, sess_test_locale)
)
predictions = pd.concat(predictions).reset_index(drop=True)
predictions.sample(5)
Out[34]:
In [35]:
def check_predictions(predictions, check_products=False):
"""
These tests need to pass as they will also be applied on the evaluator
"""
test_locale_names = test_sessions['locale'].unique()
for locale in test_locale_names:
sess_test = test_sessions.query(f'locale == "{locale}"')
preds_locale = predictions[predictions['locale'] == sess_test['locale'].iloc[0]]
assert sorted(preds_locale.index.values) == sorted(sess_test.index.values), f"Session ids of {locale} doesn't match"
if check_products:
# This check is not done on the evaluator
# but you can run it to verify there is no mixing of products between locales
# Since the ground truth next item will always belong to the same locale
# Warning - This can be slow to run
products = read_product_data().query(f'locale == "{locale}"')
predicted_products = np.unique( np.array(list(preds_locale["next_item_prediction"].values)) )
assert np.all( np.isin(predicted_products, products['id']) ), f"Invalid products in {locale} predictions"
In [36]:
check_predictions(predictions)
# Its important that the parquet file you submit is saved with pyarrow backend
predictions.to_parquet(f'submission_{task}.parquet', engine='pyarrow')
In [ ]:
# 读取产品数据的前100行
products = pd.read_csv('products_train.csv', nrows=100)
In [38]:
# 针对每个会话进行预测
predictions = []
for index, row in test_sessions.iterrows():
session = row['prev_items']
locale = row['locale']
locale_products = products[products['locale'] == locale]
if not locale_products.empty:
predicted_products = predict_next_products(session, locale_products)
predictions.append(predicted_products)
else:
predictions.append([]) # If no matching products, append an empty list
# 创建预测结果DataFrame
output_data = pd.DataFrame({'next_item': predictions})
# 将预测结果存储为parquet文件
output_data.to_parquet('predictions.parquet', index=False)
In [42]:
import pandas as pd
# 从Parquet文件读取数据
predictions = pd.read_parquet('predictions.parquet')
# 打印数据前几行
print(predictions.head(100))
In [17]:
!aicrowd submission create -c task-1-next-product-recommendation -f "submission_task1.parquet"
Content
Comments
You must login before you can post a comment.