How to Double Your Revenue from Web Scraping with Python

import re

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium_python import smartproxy
import json

def scraper():
    chrome_options = Options()
    chrome_options.add_argument('--disable-gpu')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_argument('--disable-dev-shm-usage')
    desired_capabilities = smartproxy()

    driver = webdriver.Chrome('selenium-scraper/chromedriver', options=chrome_options, desired_capabilities=desired_capabilities)
    items_xpath = "//div[starts-with(@id, '100_dealView')]"
    driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn&nocache=1569852387822')

    elems = driver.find_elements_by_xpath(items_xpath)

    items_list = []
    single_digit_deal = re.compile(r'\d% off')
    double_digit_deal = re.compile(r'\d% off')

    for i, item in enumerate(elems):
        off_match = single_digit_deal.findall(item.text)
        if not off_match:
            off_match = double_digit_deal.findall(item.text)
            if off_match is not None:
                strip_data(i, item.text)
        else:
            strip_data(i, item.text)


def strip_data(item_index, data):

    items = {}

    price_regex = re.compile(r'\$\d{1,10}.\d{1,2}')
    price = price_regex.findall(data)

    # assuming those with price are legit
    if price:
        # account for price range
        if len(price) == 1:
            # Ignore random one price items
            new_price = None
        elif len(price) <= 2:
            new_price = price[0]
            list_price = price[1]
        elif len(price) > 2:
            # price range gotcha
            new_price = f"{price[0]} - {price[1]}"
            list_price = f"{price[2]} - {price[3]}"

        # check for new_price only then proceed
        if new_price:
            if "Ends in" not in data:
                title_slice = data.split('\n')[2]
                # Check for misplaced items check for basic length > 10
                if len(title_slice) >= 10:
                    product_title = title_slice
                else:
                    return

                items[item_index] = {}
                items[item_index]['product_title'] = product_title
                items[item_index]['new_price'] = new_price
                items[item_index]['list_price'] = list_price

                with open('products.json', 'w') as f:
                    f.write(json.dumps(items, sort_keys=True, indent=4))



if __name__ == '__main__':
    scraper()