Creating an AI Trading Bot using Machine Learning with help of AI¶

import os
import alpaca_trade_api as tradeapi
import pandas as pd
from datetime import datetime, timedelta
import time

# For Paper Trading
os.environ['APCA_API_BASE_URL'] = 'https://paper-api.alpaca.markets'
# Replace with your actual keys or set them as environment variables
os.environ['APCA_API_KEY_ID'] = 'your API KEY'
os.environ['APCA_API_SECRET_KEY'] = 'your SECRET KEY'

API_KEY = os.getenv('APCA_API_KEY_ID')
API_SECRET = os.getenv('APCA_API_SECRET_KEY')
BASE_URL = os.getenv('APCA_API_BASE_URL')

api = tradeapi.REST(API_KEY, API_SECRET, BASE_URL, api_version='v2')

print("Connected to Alpaca Paper Trading.")
# Check account
# account = api.get_account()
# print(f"Account status: {account.status}")

def fetch_data(symbol, timeframe, start_date_str, end_date_str=None):
    """Fetches historical crypto data from Alpaca."""
    if end_date_str is None:
        end_date_str = datetime.now().strftime('%Y-%m-%d')

    # Alpaca API expects ISO 8601 format for start/end times
    # And it has a limit on how many bars can be fetched per request (e.g., 10000 for crypto)
    # So we may need to fetch in chunks if requesting a long period.

    all_bars = []
    start_dt = pd.to_datetime(start_date_str, utc=True)
    end_dt = pd.to_datetime(end_date_str, utc=True)

    # Fetch data in chunks to avoid hitting API limits for very long periods
    # For 1-minute data, 10000 bars is about 7 days.
    # Let's fetch data in smaller chunks, e.g., 5 days at a time.
    current_start = start_dt
    while current_start < end_dt:
        chunk_end = min(current_start + timedelta(days=5), end_dt) # Adjust chunk size as needed
        print(f"Fetching data from {current_start.isoformat()} to {chunk_end.isoformat()}")

        # Alpaca's get_crypto_bars expects start and end in ISO format
        bars = api.get_crypto_bars(
            symbol,
            timeframe,
            start=current_start.isoformat(),
            end=chunk_end.isoformat(),
            limit=10000 # Max limit per request
        ).df

        if bars.empty:
            print(f"No data found for chunk starting {current_start.isoformat()}")
            if current_start + timedelta(days=5) > end_dt and not all_bars: # if first chunk and no data
                 break
            current_start += timedelta(days=5) # Move to next chunk period
            time.sleep(1) # Be nice to the API
            continue

        all_bars.append(bars)
        # Update current_start to the timestamp of the last bar fetched + 1 minute
        # to avoid overlap and to ensure we move forward.
        if not bars.index.empty:
            current_start = bars.index[-1] + pd.Timedelta(minutes=1)
        else: # Should not happen if bars is not empty, but as a safeguard
            current_start += timedelta(days=5)

        time.sleep(1) # Respect API rate limits

    if not all_bars:
        print("No data fetched. Check your date range or symbol.")
        return pd.DataFrame()

    df = pd.concat(all_bars)
    df = df[~df.index.duplicated(keep='first')] # Remove potential duplicates from chunking
    df = df.sort_index() # Ensure chronological order
    df = df[df.index <= end_dt] # Ensure we don't go past the requested end_dt
    return df

# Example usage:
symbol = "BTC/USD" # Alpaca uses "BTC/USD" for crypto pairs
timeframe = tradeapi. টাইমফ্রেম.মিনিট # or "1Min" for older SDK versions
# Fetch last 30 days of data for example
start_date = (datetime.now() - timedelta(days=30)).strftime('%Y-%m-%d')
# end_date = datetime.now().strftime('%Y-%m-%d') # Fetches up to now

btc_data = fetch_data(symbol, timeframe, start_date)
if not btc_data.empty:
    print(f"Fetched {len(btc_data)} rows of {symbol} data.")
    print(btc_data.head())
    print(btc_data.tail())
else:
    print("Failed to fetch data.")

import os
import alpaca_trade_api as tradeapi
import pandas as pd
from datetime import datetime, timedelta
import time

# For Paper Trading
os.environ['APCA_API_BASE_URL'] = 'https://paper-api.alpaca.markets'
# Replace with your actual keys or set them as environment variables
# Make sure these are set correctly if you haven't already
os.environ['APCA_API_KEY_ID'] = 'your API KEY'
os.environ['APCA_API_SECRET_KEY'] = 'your SECRET KEY'
# os.environ['APCA_API_KEY_ID'] = 'YOUR_PAPER_API_KEY_ID'
# os.environ['APCA_API_SECRET_KEY'] = 'YOUR_PAPER_API_SECRET_KEY'

API_KEY = os.getenv('APCA_API_KEY_ID')
API_SECRET = os.getenv('APCA_API_SECRET_KEY')
BASE_URL = os.getenv('APCA_API_BASE_URL')

# Ensure API_KEY and API_SECRET are not None
if not API_KEY or not API_SECRET:
    raise ValueError("API_KEY or API_SECRET not set. Please set them as environment variables or directly in the script.")

api = tradeapi.REST(API_KEY, API_SECRET, BASE_URL, api_version='v2')

print("Attempting to connect to Alpaca Paper Trading...")
try:
    account = api.get_account()
    print(f"Successfully connected. Account status: {account.status}")
except Exception as e:
    print(f"Failed to connect or get account info: {e}")
    # Exit or handle if connection fails
    exit()


def fetch_data(symbol, timeframe_enum, start_date_str, end_date_str=None): # Changed timeframe to timeframe_enum
    """Fetches historical crypto data from Alpaca."""
    if end_date_str is None:
        end_date_str = datetime.now().strftime('%Y-%m-%d')

    all_bars = []
    start_dt = pd.to_datetime(start_date_str, utc=True)
    end_dt = pd.to_datetime(end_date_str, utc=True)

    current_start = start_dt
    while current_start < end_dt:
        # Calculate chunk_end, ensuring it doesn't exceed end_dt
        # For 1-minute data, 10000 bars is approx 6.94 days. Let's use 6 days to be safe.
        chunk_end_candidate = current_start + timedelta(days=6)
        chunk_end = min(chunk_end_candidate, end_dt)

        print(f"Fetching data from {current_start.isoformat()} to {chunk_end.isoformat()}")

        try:
            bars = api.get_crypto_bars(
                symbol,
                timeframe_enum, # Use the passed enum
                start=current_start.isoformat(),
                end=chunk_end.isoformat(), # Ensure end is also passed
                limit=10000
            ).df
        except Exception as e:
            print(f"Error fetching data chunk: {e}")
            # Decide how to handle: break, retry, or skip chunk
            current_start = chunk_end # Move to next potential period
            time.sleep(5) # Wait longer if an error occurred
            continue


        if bars.empty:
            print(f"No data found for chunk starting {current_start.isoformat()}")
            if current_start >= end_dt and not all_bars: # if first chunk and no data
                 break
            current_start = chunk_end # Move to next chunk period
            time.sleep(1) # Be nice to the API
            continue

        all_bars.append(bars)
        if not bars.index.empty:
            # Move current_start to the timestamp of the last bar fetched + 1 unit of timeframe
            # For TimeFrame.Minute, this is +1 minute
            if timeframe_enum == tradeapi.TimeFrame.Minute:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(minutes=1)
            elif timeframe_enum == tradeapi.TimeFrame.Hour:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(hours=1)
            # Add other timeframes if needed
            else: # Default for Day or others
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(days=1)

            # Ensure current_start does not go beyond end_dt excessively in the loop condition
            if current_start > end_dt and chunk_end_candidate >= end_dt :
                break
        else: # Should not happen if bars is not empty, but as a safeguard
            current_start = chunk_end

        time.sleep(1) # Respect API rate limits

    if not all_bars:
        print("No data fetched. Check your date range, symbol, or API connection.")
        return pd.DataFrame()

    df = pd.concat(all_bars)
    df = df[~df.index.duplicated(keep='first')] # Remove potential duplicates
    df = df.sort_index() # Ensure chronological order

    # Filter to ensure we are within the originally requested start_dt and end_dt precisely
    df = df[(df.index >= start_dt) & (df.index <= end_dt)]
    return df

# Example usage:
symbol = "BTC/USD" # Alpaca uses "BTC/USD" for crypto pairs
# CORRECTED TIMEFRAME:
timeframe_to_use = tradeapi.TimeFrame.Minute # Corrected
# Alternative: timeframe_to_use = "1Min" (as a string, for some SDK versions or if TimeFrame enum is problematic)

# Fetch last 7 days of data for example (a smaller range for quicker testing)
start_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
# end_date = datetime.now().strftime('%Y-%m-%d') # Fetches up to now by default if None

btc_data = fetch_data(symbol, timeframe_to_use, start_date)

if not btc_data.empty:
    print(f"\nFetched {len(btc_data)} rows of {symbol} data.")
    print("Sample data (first 5 rows):")
    print(btc_data.head())
    print("\nSample data (last 5 rows):")
    print(btc_data.tail())
else:
    print("Failed to fetch data or no data available for the period.")

import pandas_ta as ta

def create_features(df):
    if df.empty or 'close' not in df.columns:
        print("DataFrame is empty or 'close' column is missing.")
        return df

    df.ta.sma(length=10, append=True, col_names=('SMA_10'))
    df.ta.sma(length=30, append=True, col_names=('SMA_30'))
    df.ta.ema(length=10, append=True, col_names=('EMA_10'))
    df.ta.rsi(length=14, append=True, col_names=('RSI_14'))
    df.ta.macd(append=True, col_names=('MACD_12_26_9', 'MACDh_12_26_9', 'MACDs_12_26_9'))
    df.ta.bbands(length=20, append=True, col_names=('BBL_20_2.0', 'BBM_20_2.0', 'BBU_20_2.0', 'BBB_20_2.0', 'BBP_20_2.0'))
    df.ta.atr(length=14, append=True, col_names=('ATR_14'))

    # Lagged returns
    for lag in [1, 3, 5, 10]:
        df[f'return_{lag}m'] = df['close'].pct_change(periods=lag)

    # Add more features: volatility, momentum, volume-based if available, etc.
    # e.g., log returns, price relative to moving average, etc.

    df.dropna(inplace=True) # Remove rows with NaNs created by indicators
    return df

if not btc_data.empty:
    featured_data = create_features(btc_data.copy()) # Use .copy() to avoid modifying original
    print("\nData with features:")
    print(featured_data.head())
else:
    print("Cannot create features, btc_data is empty.")
    featured_data = pd.DataFrame() # ensure it's a DataFrame

import os
import alpaca_trade_api as tradeapi
import pandas as pd
from datetime import datetime, timedelta
import time

# For Paper Trading
os.environ['APCA_API_BASE_URL'] = 'https://paper-api.alpaca.markets'
# Replace with your actual keys or set them as environment variables
# Make sure these are set correctly if you haven't already
os.environ['APCA_API_KEY_ID'] = 'YOUR_PAPER_API_KEY_ID'
os.environ['APCA_API_SECRET_KEY'] = 'YOUR_PAPER_API_SECRET_KEY'
# os.environ['APCA_API_KEY_ID'] = 'YOUR_PAPER_API_KEY_ID'
# os.environ['APCA_API_SECRET_KEY'] = 'YOUR_PAPER_API_SECRET_KEY'

API_KEY = os.getenv('APCA_API_KEY_ID')
API_SECRET = os.getenv('APCA_API_SECRET_KEY')
BASE_URL = os.getenv('APCA_API_BASE_URL')

# Ensure API_KEY and API_SECRET are not None
if not API_KEY or not API_SECRET:
    raise ValueError("API_KEY or API_SECRET not set. Please set them as environment variables or directly in the script.")

api = tradeapi.REST(API_KEY, API_SECRET, BASE_URL, api_version='v2')

print("Attempting to connect to Alpaca Paper Trading...")
try:
    account = api.get_account()
    print(f"Successfully connected. Account status: {account.status}")
except Exception as e:
    print(f"Failed to connect or get account info: {e}")
    # Exit or handle if connection fails
    exit()


def fetch_data(symbol, timeframe_enum, start_date_str, end_date_str=None): # Changed timeframe to timeframe_enum
    """Fetches historical crypto data from Alpaca."""
    if end_date_str is None:
        end_date_str = datetime.now().strftime('%Y-%m-%d')

    all_bars = []
    start_dt = pd.to_datetime(start_date_str, utc=True)
    end_dt = pd.to_datetime(end_date_str, utc=True)

    current_start = start_dt
    while current_start < end_dt:
        # Calculate chunk_end, ensuring it doesn't exceed end_dt
        # For 1-minute data, 10000 bars is approx 6.94 days. Let's use 6 days to be safe.
        chunk_end_candidate = current_start + timedelta(days=6)
        chunk_end = min(chunk_end_candidate, end_dt)

        print(f"Fetching data from {current_start.isoformat()} to {chunk_end.isoformat()}")

        try:
            bars = api.get_crypto_bars(
                symbol,
                timeframe_enum, # Use the passed enum
                start=current_start.isoformat(),
                end=chunk_end.isoformat(), # Ensure end is also passed
                limit=10000
            ).df
        except Exception as e:
            print(f"Error fetching data chunk: {e}")
            # Decide how to handle: break, retry, or skip chunk
            current_start = chunk_end # Move to next potential period
            time.sleep(5) # Wait longer if an error occurred
            continue


        if bars.empty:
            print(f"No data found for chunk starting {current_start.isoformat()}")
            if current_start >= end_dt and not all_bars: # if first chunk and no data
                 break
            current_start = chunk_end # Move to next chunk period
            time.sleep(1) # Be nice to the API
            continue

        all_bars.append(bars)
        if not bars.index.empty:
            # Move current_start to the timestamp of the last bar fetched + 1 unit of timeframe
            # For TimeFrame.Minute, this is +1 minute
            if timeframe_enum == tradeapi.TimeFrame.Minute:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(minutes=1)
            elif timeframe_enum == tradeapi.TimeFrame.Hour:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(hours=1)
            # Add other timeframes if needed
            else: # Default for Day or others
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(days=1)

            # Ensure current_start does not go beyond end_dt excessively in the loop condition
            if current_start > end_dt and chunk_end_candidate >= end_dt :
                break
        else: # Should not happen if bars is not empty, but as a safeguard
            current_start = chunk_end

        time.sleep(1) # Respect API rate limits

    if not all_bars:
        print("No data fetched. Check your date range, symbol, or API connection.")
        return pd.DataFrame()

    df = pd.concat(all_bars)
    df = df[~df.index.duplicated(keep='first')] # Remove potential duplicates
    df = df.sort_index() # Ensure chronological order

    # Filter to ensure we are within the originally requested start_dt and end_dt precisely
    df = df[(df.index >= start_dt) & (df.index <= end_dt)]
    return df

# Example usage:
symbol = "BTC/USD" # Alpaca uses "BTC/USD" for crypto pairs
# CORRECTED TIMEFRAME:
timeframe_to_use = tradeapi.TimeFrame.Minute # Corrected
# Alternative: timeframe_to_use = "1Min" (as a string, for some SDK versions or if TimeFrame enum is problematic)

# Fetch last 7 days of data for example (a smaller range for quicker testing)
start_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')
# end_date = datetime.now().strftime('%Y-%m-%d') # Fetches up to now by default if None

btc_data = fetch_data(symbol, timeframe_to_use, start_date)

if not btc_data.empty:
    print(f"\nFetched {len(btc_data)} rows of {symbol} data.")
    print("Sample data (first 5 rows):")
    print(btc_data.head())
    print("\nSample data (last 5 rows):")
    print(btc_data.tail())
else:
    print("Failed to fetch data or no data available for the period.")

pip install finta

import pandas as pd
from finta import TA # Import the TA module from finta
import numpy as np # For np.nan if needed for BBP/BBB calculation edge cases

# This is your existing GOOD CODE (data fetching part)
# I'm including it here just for completeness of the script context
# ... (paste your GOOD CODE here, from 'import os' down to the btc_data fetching and printing)
# For brevity, I'll assume btc_data is already fetched as per your GOOD CODE.

# --- MODIFIED create_features FUNCTION USING finta ---
def create_features_finta(df):
    if df.empty or 'close' not in df.columns:
        print("DataFrame is empty or 'close' column is missing.")
        return df.copy() # Return a copy to avoid modifying original if it's returned early

    # Ensure required columns for some indicators are present
    required_ohlc = ['open', 'high', 'low', 'close']
    if not all(col in df.columns for col in required_ohlc):
        print(f"DataFrame is missing one or more required columns: {required_ohlc}")
        # Attempt to continue if only 'close' is needed for some indicators,
        # but ATR and BBands might fail or need to be skipped.
        # For now, let's be strict for this example.
        # You might want to add more nuanced handling later.
        # return df.copy() # Or raise an error

    # SMA (Simple Moving Average)
    df['SMA_10'] = TA.SMA(df, period=10, column='close')
    df['SMA_30'] = TA.SMA(df, period=30, column='close')

    # EMA (Exponential Moving Average)
    df['EMA_10'] = TA.EMA(df, period=10, column='close') # finta's EMA defaults to 'close' if not specified

    # RSI (Relative Strength Index)
    df['RSI_14'] = TA.RSI(df, period=14, column='close')

    # MACD (Moving Average Convergence Divergence)
    # TA.MACD returns a DataFrame with 'MACD' and 'SIGNAL' columns
    macd_df = TA.MACD(df, period_fast=12, period_slow=26, signal=9, column='close')
    df['MACD_12_26_9'] = macd_df['MACD']
    df['MACDs_12_26_9'] = macd_df['SIGNAL'] # This is the signal line
    df['MACDh_12_26_9'] = df['MACD_12_26_9'] - df['MACDs_12_26_9'] # Histogram is MACD - Signal

    # Bollinger Bands (BBANDS)
    # TA.BBANDS returns a DataFrame with 'BB_UPPER', 'BB_MIDDLE', 'BB_LOWER'
    # It uses 'high', 'low', 'close' columns by default.
    if all(col in df.columns for col in ['high', 'low', 'close']):
        bb_df = TA.BBANDS(df, period=20, std_multiplier=2.0) # std_multiplier is for standard deviations
        df['BBL_20_2.0'] = bb_df['BB_LOWER']
        df['BBM_20_2.0'] = bb_df['BB_MIDDLE'] # This is the SMA(20)
        df['BBU_20_2.0'] = bb_df['BB_UPPER']

        # Calculate BBB (Bollinger Band Width) and BBP (Bollinger Band Percent B) manually
        # BBB = (Upper Band - Lower Band) / Middle Band
        df['BBB_20_2.0'] = (df['BBU_20_2.0'] - df['BBL_20_2.0']) / df['BBM_20_2.0']
        # BBP = (Price - Lower Band) / (Upper Band - Lower Band)
        # Handle division by zero if Upper Band == Lower Band (can happen in flat markets or with very short periods)
        band_range = df['BBU_20_2.0'] - df['BBL_20_2.0']
        df['BBP_20_2.0'] = np.where(band_range == 0, 0.5, (df['close'] - df['BBL_20_2.0']) / band_range) # if range is 0, BBP is undefined, set to 0.5 (middle) or NaN
    else:
        print("Skipping Bollinger Bands calculation as 'high', 'low', 'close' columns are not all present.")
        for col_name in ['BBL_20_2.0', 'BBM_20_2.0', 'BBU_20_2.0', 'BBB_20_2.0', 'BBP_20_2.0']:
            df[col_name] = np.nan


    # ATR (Average True Range)
    # TA.ATR requires 'high', 'low', 'close' columns.
    if all(col in df.columns for col in ['high', 'low', 'close']):
        df['ATR_14'] = TA.ATR(df, period=14)
    else:
        print("Skipping ATR calculation as 'high', 'low', 'close' columns are not all present.")
        df['ATR_14'] = np.nan


    # Lagged returns (this part is standard pandas, no change needed)
    for lag in [1, 3, 5, 10]:
        df[f'return_{lag}m'] = df['close'].pct_change(periods=lag)

    df.dropna(inplace=True) # Remove rows with NaNs created by indicators
    return df

# --- End of MODIFIED create_features FUNCTION ---

# Your existing code to call the function (make sure btc_data is defined from GOOD CODE)
if 'btc_data' in locals() and not btc_data.empty:
    # It's good practice to lowercase column names if the library expects it,
    # though Alpaca's .df usually does this. Let's be explicit.
    btc_data.columns = [col.lower() for col in btc_data.columns]
    
    featured_data = create_features_finta(btc_data.copy()) # Use .copy() to avoid modifying original
    print("\nData with features (using finta):")
    if not featured_data.empty:
        print(featured_data.head())
        print(f"\nNumber of rows in featured_data: {len(featured_data)}")
    else:
        print("Featured data is empty after processing.")
elif 'btc_data' in locals() and btc_data.empty:
    print("Cannot create features, btc_data is empty.")
    featured_data = pd.DataFrame() # ensure it's a DataFrame
else:
    print("btc_data not found. Please run the data fetching part first.")
    featured_data = pd.DataFrame()

import os
import alpaca_trade_api as tradeapi
import pandas as pd
from datetime import datetime, timedelta
import time
from finta import TA # Import for technical indicators
import numpy as np   # For np.nan

# For Paper Trading
os.environ['APCA_API_BASE_URL'] = 'https://paper-api.alpaca.markets'
# Replace with your actual keys or set them as environment variables
# Make sure these are set correctly if you haven't already
# IMPORTANT: Replace with your actual keys for testing
#os.environ['APCA_API_KEY_ID'] = 'YOUR_PAPER_API_KEY_ID' # <--- REPLACE
#os.environ['APCA_API_SECRET_KEY'] = 'YOUR_PAPER_API_SECRET_KEY' # <--- REPLACE
os.environ['APCA_API_KEY_ID'] = 'YOUR_PAPER_API_KEY_ID'
os.environ['APCA_API_SECRET_KEY'] = 'YOUR_PAPER_API_SECRET_KEY'

API_KEY = os.getenv('APCA_API_KEY_ID')
API_SECRET = os.getenv('APCA_API_SECRET_KEY')
BASE_URL = os.getenv('APCA_API_BASE_URL')

# Ensure API_KEY and API_SECRET are not None
if not API_KEY or not API_SECRET or API_KEY == 'YOUR_PAPER_API_KEY_ID':
    raise ValueError("API_KEY or API_SECRET not set or still default. Please set them as environment variables or directly in the script.")

api = tradeapi.REST(API_KEY, API_SECRET, BASE_URL, api_version='v2')

print("Attempting to connect to Alpaca Paper Trading...")
try:
    account = api.get_account()
    print(f"Successfully connected. Account status: {account.status}")
except Exception as e:
    print(f"Failed to connect or get account info: {e}")
    exit()


def fetch_data(symbol, timeframe_enum, start_date_str, end_date_str=None):
    if end_date_str is None:
        end_date_str = datetime.now().strftime('%Y-%m-%d')

    all_bars = []
    start_dt = pd.to_datetime(start_date_str, utc=True)
    end_dt = pd.to_datetime(end_date_str, utc=True)

    current_start = start_dt
    while current_start < end_dt:
        chunk_end_candidate = current_start + timedelta(days=6)
        chunk_end = min(chunk_end_candidate, end_dt)
        print(f"Fetching data from {current_start.isoformat()} to {chunk_end.isoformat()}")
        try:
            bars = api.get_crypto_bars(
                symbol,
                timeframe_enum,
                start=current_start.isoformat(),
                end=chunk_end.isoformat(),
                limit=10000
            ).df
        except Exception as e:
            print(f"Error fetching data chunk: {e}")
            current_start = chunk_end
            time.sleep(5)
            continue

        if bars.empty:
            print(f"No data found for chunk starting {current_start.isoformat()}")
            if current_start >= end_dt and not all_bars:
                 break
            current_start = chunk_end
            time.sleep(1)
            continue

        all_bars.append(bars)
        if not bars.index.empty:
            if timeframe_enum == tradeapi.TimeFrame.Minute:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(minutes=1)
            elif timeframe_enum == tradeapi.TimeFrame.Hour:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(hours=1)
            else:
                 current_start = bars.index[-1].to_pydatetime() + pd.Timedelta(days=1)
            if current_start > end_dt and chunk_end_candidate >= end_dt :
                break
        else:
            current_start = chunk_end
        time.sleep(1)

    if not all_bars:
        print("No data fetched. Check your date range, symbol, or API connection.")
        return pd.DataFrame()

    df = pd.concat(all_bars)
    df = df[~df.index.duplicated(keep='first')]
    df = df.sort_index()
    df = df[(df.index >= start_dt) & (df.index <= end_dt)]
    return df

# Example usage:
symbol = "BTC/USD"
timeframe_to_use = tradeapi.TimeFrame.Minute
start_date = (datetime.now() - timedelta(days=7)).strftime('%Y-%m-%d')

btc_data = fetch_data(symbol, timeframe_to_use, start_date)

if not btc_data.empty:
    print(f"\nFetched {len(btc_data)} rows of {symbol} data.")
    print("Sample data (first 5 rows):")
    print(btc_data.head())
    print("\nSample data (last 5 rows):")
    print(btc_data.tail())
    # Ensure column names are lowercase for finta
    btc_data.columns = [col.lower() for col in btc_data.columns]
else:
    print("Failed to fetch data or no data available for the period.")
    btc_data = pd.DataFrame() # ensure it's a DataFrame for subsequent steps


# --- REVISED create_features FUNCTION USING finta ---
def create_features(df_input): # Renamed input to df_input to avoid clash with outer scope df
    df = df_input.copy() # Work on a copy

    if df.empty or 'close' not in df.columns:
        print("DataFrame is empty or 'close' column is missing.")
        return df # Return the empty or deficient (but copied) DataFrame

    # Ensure required columns for some indicators are present
    required_ohlc = ['open', 'high', 'low', 'close']
    has_ohlc = all(col in df.columns for col in required_ohlc)

    # SMA (Simple Moving Average)
    df['SMA_10'] = TA.SMA(df, period=10, column='close')
    df['SMA_30'] = TA.SMA(df, period=30, column='close')

    # EMA (Exponential Moving Average)
    df['EMA_10'] = TA.EMA(df, period=10, column='close')

    # RSI (Relative Strength Index)
    df['RSI_14'] = TA.RSI(df, period=14, column='close')

    # MACD (Moving Average Convergence Divergence)
    macd_df = TA.MACD(df, period_fast=12, period_slow=26, signal=9, column='close')
    df['MACD_12_26_9'] = macd_df['MACD']
    df['MACDs_12_26_9'] = macd_df['SIGNAL']
    df['MACDh_12_26_9'] = df['MACD_12_26_9'] - df['MACDs_12_26_9']

    # Bollinger Bands (BBANDS)
    if has_ohlc:
        bb_df = TA.BBANDS(df, period=20, std_multiplier=2.0)
        df['BBL_20_2.0'] = bb_df['BB_LOWER']
        df['BBM_20_2.0'] = bb_df['BB_MIDDLE']
        df['BBU_20_2.0'] = bb_df['BB_UPPER']
        df['BBB_20_2.0'] = (df['BBU_20_2.0'] - df['BBL_20_2.0']) / df['BBM_20_2.0']
        band_range = df['BBU_20_2.0'] - df['BBL_20_2.0']
        df['BBP_20_2.0'] = np.where(band_range == 0, 0.5, (df['close'] - df['BBL_20_2.0']) / band_range)
    else:
        print("Skipping Bollinger Bands calculation as 'open', 'high', 'low', 'close' columns are not all present.")
        for col_name in ['BBL_20_2.0', 'BBM_20_2.0', 'BBU_20_2.0', 'BBB_20_2.0', 'BBP_20_2.0']:
            df[col_name] = np.nan

    # ATR (Average True Range)
    if has_ohlc:
        df['ATR_14'] = TA.ATR(df, period=14)
    else:
        print("Skipping ATR calculation as 'open', 'high', 'low', 'close' columns are not all present.")
        df['ATR_14'] = np.nan

    # Lagged returns
    for lag in [1, 3, 5, 10]:
        df[f'return_{lag}m'] = df['close'].pct_change(periods=lag)

    df.dropna(inplace=True)
    return df
# --- End of REVISED create_features FUNCTION ---


# Call the feature creation function
if not btc_data.empty:
    featured_data = create_features(btc_data) # No need for .copy() here as create_features does it
    print("\nData with features (using finta):")
    if not featured_data.empty:
        print(featured_data.head())
        print(f"\nNumber of rows in featured_data: {len(featured_data)}")
        print("\nColumns in featured_data:")
        print(featured_data.columns.tolist())
    else:
        print("Featured data is empty after processing and NaN removal.")
else:
    print("Cannot create features, btc_data is empty.")
    featured_data = pd.DataFrame()

import pandas as pd
from sklearn.model_selection import train_test_split, TimeSeriesSplit # TimeSeriesSplit for more advanced CV
from sklearn.ensemble import RandomForestClassifier # Example alternative model
from sklearn.metrics import accuracy_score, classification_report
import xgboost as xgb
import numpy as np # For np.nan and other numpy operations if needed

def prepare_data_for_model(df_input, target_horizon=5, test_size=0.2):
    """
    Prepares X (features) and y (target) for the ML model.
    Target: 1 if price increases after target_horizon periods, 0 otherwise.
    """
    df = df_input.copy() # Work on a copy to avoid modifying the original DataFrame

    if df.empty or 'close' not in df.columns:
        print("DataFrame is empty or 'close' column is missing for target creation.")
        return pd.DataFrame(), pd.DataFrame(), pd.Series(dtype='float64'), pd.Series(dtype='float64'), []

    df['target'] = (df['close'].shift(-target_horizon) > df['close']).astype(int)
    df.dropna(inplace=True)

    # Identify feature columns:
    # ----- THIS IS THE KEY CHANGE -----
    cols_to_exclude = ['open', 'high', 'low', 'close', 'volume', # Raw data
                       'trade_count', 'vwap',                     # Other raw data (if present)
                       'future_price', 'price_change_pct', 'target', # Target-related
                       'symbol']                                 # <<< ADD 'symbol' HERE

    # Filter out columns that might not exist in df to avoid errors, then exclude
    actual_cols_to_exclude = [col for col in cols_to_exclude if col in df.columns]
    
    feature_columns = [col for col in df.columns if col not in actual_cols_to_exclude]

    if not feature_columns:
        print("No feature columns found after exclusion. Check exclusion list and DataFrame contents.")
        return pd.DataFrame(), pd.DataFrame(), pd.Series(dtype='float64'), pd.Series(dtype='float64'), []
        
    X = df[feature_columns]
    y = df['target']

    if X.empty or y.empty:
        print("X or y is empty after processing. Not enough data.")
        return pd.DataFrame(), pd.DataFrame(), pd.Series(dtype='float64'), pd.Series(dtype='float64'), feature_columns

    split_index = int(len(X) * (1 - test_size))
    X_train, X_test = X.iloc[:split_index], X.iloc[split_index:]
    y_train, y_test = y.iloc[:split_index], y.iloc[split_index:]

    return X_train, X_test, y_train, y_test, feature_columns


# --- Main script execution for Step 4 ---
# This assumes 'featured_data' DataFrame is available from the previous step
model = None # Initialize model variable

if 'featured_data' in locals() and not featured_data.empty:
    # Ensure 'close' column exists if not already checked by create_features
    if 'close' not in featured_data.columns:
        print("Error: 'close' column is missing in featured_data. Cannot proceed with model training.")
    else:
        X_train, X_test, y_train, y_test, feature_cols = prepare_data_for_model(
            featured_data, # No .copy() needed here as prepare_data_for_model does it
            target_horizon=5, # Predict 5 minutes into the future
            test_size=0.2     # 20% of data for testing
        )

        if not X_train.empty and not y_train.empty and not X_test.empty and not y_test.empty:
            print(f"\nTraining data shape: X_train: {X_train.shape}, y_train: {y_train.shape}")
            print(f"Test data shape: X_test: {X_test.shape}, y_test: {y_test.shape}")
            print(f"Number of features: {len(feature_cols)}")
            # print(f"Features used: {feature_cols}") # Can be very long, print first few if needed
            print(f"First 5 features: {feature_cols[:5]}")


            # Example Model: XGBoost (often performs well)
            # For XGBoost versions < 1.6, you might need use_label_encoder=False if y_train is a Pandas Series
            # For XGBoost versions >= 1.6, use_label_encoder is deprecated/removed.
            # If you encounter issues, ensure y_train is a NumPy array: y_train.to_numpy()
            model = xgb.XGBClassifier(
                objective='binary:logistic', # For binary up/down prediction
                n_estimators=100,
                learning_rate=0.1,
                max_depth=3,
                # use_label_encoder=False, # Removed for better compatibility with XGBoost >= 1.6
                eval_metric='logloss'      # Evaluation metric for binary classification
            )

            print("\nTraining the XGBoost model...")
            try:
                model.fit(X_train, y_train) # XGBoost can often handle pandas DataFrames/Series directly

                # Evaluate on test set
                y_pred = model.predict(X_test)
                y_pred_proba = model.predict_proba(X_test)[:, 1] # Probabilities for the positive class

                print("\nModel Evaluation on Test Set:")
                print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
                
                # Check unique classes in y_test and y_pred for classification_report
                # This helps avoid errors if one class is missing in predictions or test set after split/dropna
                report = classification_report(y_test, y_pred, zero_division=0)
                print(report)

                # Feature importance (for tree-based models like XGBoost)
                if hasattr(model, 'feature_importances_'):
                    importances = pd.Series(model.feature_importances_, index=X_train.columns).sort_values(ascending=False)
                    print("\nTop 10 Feature Importances:")
                    print(importances.head(10))

            except Exception as e:
                print(f"An error occurred during model training or evaluation: {e}")
                model = None # Ensure model is None if training failed

        else:
            print("Not enough data to create training/test sets after feature engineering and target creation.")
            if 'feature_cols' in locals() and feature_cols: # Check if feature_cols was populated
                 print(f"Identified feature columns: {feature_cols[:5]}...")
            else:
                 print("Could not identify feature columns.")

else:
    print("Cannot prepare data for model, 'featured_data' is empty or not defined.")
    # X_train, X_test, y_train, y_test, feature_cols will not be defined here