File size: 3,475 Bytes
100fb60 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import pandas as pd
import numpy as np
import os
import joblib
from preprocessing import preprocess_pipeline
def verify_data_integrity():
print("=== Starting Deep Verification ===")
# Paths
data_dir = os.path.join(os.path.dirname(__file__), '../data/crimedataset')
train_path = os.path.join(data_dir, 'train.csv')
test_path = os.path.join(data_dir, 'test.csv')
# 1. Load Data
print("\n[1] Loading Data...")
if not os.path.exists(train_path):
print("X Train file missing!")
return
df_train = pd.read_csv(train_path, parse_dates=['Dates'])
print(f"OK Train Data Loaded: {df_train.shape}")
# 2. Check for Duplicates
print("\n[2] Checking for Duplicates...")
duplicates = df_train.duplicated().sum()
if duplicates > 0:
print(f"! Warning: {duplicates} duplicate rows found in training data.")
else:
print("OK No duplicates found.")
# 3. Class Balance
print("\n[3] Checking Class Balance...")
violent_categories = [
'ASSAULT', 'ROBBERY', 'SEX OFFENSES FORCIBLE', 'KIDNAPPING', 'HOMICIDE', 'ARSON'
]
df_train['IsViolent'] = df_train['Category'].apply(lambda x: 1 if x in violent_categories else 0)
balance = df_train['IsViolent'].value_counts(normalize=True)
print(f"Violent Crime Ratio: {balance.get(1, 0)*100:.2f}%")
print(f"Non-Violent Crime Ratio: {balance.get(0, 0)*100:.2f}%")
if balance.get(1, 0) < 0.1:
print("! Severe Class Imbalance detected (<10% positive class). Model may struggle with Recall.")
# 4. Check for Data Leakage (Train vs Test overlap)
# Since test data might not have labels, we check for exact feature matches if test exists
if os.path.exists(test_path):
print("\n[4] Checking for Data Leakage (Train/Test Overlap)...")
df_test = pd.read_csv(test_path, parse_dates=['Dates'])
# Check intersection of Dates and Location
# This is a heuristic; exact row match might be too slow for large data
# We'll check a sample
train_dates = set(df_train['Dates'].dt.date.unique())
test_dates = set(df_test['Dates'].dt.date.unique())
overlap = train_dates.intersection(test_dates)
if len(overlap) > 0:
print(f"! Warning: Found {len(overlap)} days present in BOTH Train and Test sets. Possible leakage if splitting by time.")
else:
print("OK No date overlap between Train and Test.")
# 5. Verify Model Artifacts
print("\n[5] Verifying Model Artifacts...")
models_dir = os.path.join(os.path.dirname(__file__), '../models')
required_files = ['best_model.pkl', 'label_encoders.pkl', 'kmeans.pkl']
all_exist = True
for f in required_files:
fpath = os.path.join(models_dir, f)
if os.path.exists(fpath):
print(f"OK Found {f}")
# Try loading
try:
joblib.load(fpath)
print(f" -> Successfully loaded {f}")
except Exception as e:
print(f" X Failed to load {f}: {e}")
all_exist = False
else:
print(f"X Missing {f}")
all_exist = False
if all_exist:
print("\n=== Verification Complete: SYSTEM HEALTHY ===")
else:
print("\n=== Verification Complete: ISSUES DETECTED ===")
if __name__ == "__main__":
verify_data_integrity()
|