From 9f95ee14fce71065ed7d0550f56097c322fe68c0 Mon Sep 17 00:00:00 2001
From: "Bobby (aider)" <bobby@abellana.work>
Date: Sat, 8 Feb 2025 11:40:33 -0800
Subject: [PATCH] fix: Enhance data validation and debugging in ATR EMA
 screener

---
 src/screener/t_atr_ema_v2.py | 47 +++++++++++++++++++++++++++---------
 1 file changed, 35 insertions(+), 12 deletions(-)

diff --git a/src/screener/t_atr_ema_v2.py b/src/screener/t_atr_ema_v2.py
index c7a1371..7041aa1 100644
--- a/src/screener/t_atr_ema_v2.py
+++ b/src/screener/t_atr_ema_v2.py
@@ -88,32 +88,55 @@ def run_atr_ema_target_scanner(min_price: float, max_price: float, min_volume: i
                     print(f"⚠️ {ticker}: Empty DataFrame")
                     continue
                     
-                # Ensure DataFrame has required columns and proper types
-                required_columns = ['date', 'open', 'high', 'low', 'close', 'volume']
-                
-                # Print column info for debugging
+                # Debug data
                 print(f"\nProcessing {ticker}")
                 print(f"Columns present: {df.columns.tolist()}")
+                print(f"Data types: {df.dtypes}")
+                print(f"First row: {df.iloc[0]}")
+                print(f"Last row: {df.iloc[-1]}")
                 
-                # Convert columns to numeric if needed
+                # Ensure DataFrame has required columns and proper types
+                required_columns = ['date', 'open', 'high', 'low', 'close', 'volume']
+                if not all(col in df.columns for col in required_columns):
+                    missing = set(required_columns) - set(df.columns)
+                    print(f"⚠️ {ticker}: Missing columns: {missing}")
+                    continue
+                
+                # Convert columns to numeric and handle any conversion errors
                 for col in ['open', 'high', 'low', 'close', 'volume']:
-                    if col in df.columns:
+                    try:
                         df[col] = pd.to_numeric(df[col], errors='coerce')
+                    except Exception as e:
+                        print(f"⚠️ {ticker}: Error converting {col} to numeric: {str(e)}")
+                        print(f"Sample of problematic column: {df[col].head()}")
+                        raise ValueError(f"Data conversion error in {col}")
                 
-                # Verify data validity
+                # Verify data validity after conversion
                 if df['close'].isnull().any():
-                    print(f"⚠️ {ticker}: Contains null values in close price")
+                    null_rows = df[df['close'].isnull()]
+                    print(f"⚠️ {ticker}: Contains {len(null_rows)} null values in close price")
+                    print("First few null rows:")
+                    print(null_rows.head())
                     continue
                     
                 if len(df) < 50:
                     print(f"⚠️ {ticker}: Insufficient data points ({len(df)})")
                     continue
 
-                # Calculate indicator with validated data
-                results = indicator.calculate(df.copy())  # Use copy to prevent modifications
+                # Make a clean copy for indicator calculation
+                calc_df = df.copy()
                 
-                if results.empty:
-                    print(f"⚠️ {ticker}: No valid indicator results")
+                # Calculate indicator with validated data
+                try:
+                    results = indicator.calculate(calc_df)
+                    if results.empty:
+                        print(f"⚠️ {ticker}: No valid indicator results")
+                        continue
+                except Exception as e:
+                    print(f"⚠️ {ticker}: Error calculating indicator:")
+                    print(f"Error details: {str(e)}")
+                    print(f"Data shape: {calc_df.shape}")
+                    print(f"Sample data:\n{calc_df.head()}")
                     continue
 
                 results = indicator.calculate(df)