Scrape Bing Search Results

from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from extension import proxies
import json
import re
import time
from bs4 import BeautifulSoup



username = 'SPusername'
password = 'SPpassword'
endpoint = 'gate.smartproxy.com'
port = '7000'
website = 'https://www.bing.com/search?q=I%20need%20to%20throw%20a%20dinner%20party%20for%206%20people%20who%20are%20vegetarian.%20Can%20you%20suggest%20a%203-course%20menu%20with%20a%20chocolate%20dessert?&iscopilotedu=1&form=MA13G7'


def main():

    chrome_options = webdriver.ChromeOptions()

    proxies_extension = proxies(username, password, endpoint, port)

    chrome_options.add_extension(proxies_extension)

    chrome_options.add_argument("--headless=new")

    chrome = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)

    chrome.get(website)

    html = chrome.page_source

    time.sleep(2)

    soup = BeautifulSoup(html, "html.parser")

    # Find the <script> tag that contains the JavaScript object
    script_tag = soup.find('script', text=lambda text: text and 'var sj_appHTML' in text)

    # Extract the JavaScript object from the <script> tag
    js_text = script_tag.text.strip().split('=', 1)[1].strip().rstrip(';')
    
    # Extract JSON

    match = re.search(r'_w\[\"_sydCachedString\"] =\"(.*?)\";', js_text)

    json_str = match.group(1)

    decoded_string = json_str.encode('utf-8').decode('unicode_escape')

    data = json.loads(decoded_string)

    with open('bing.json', 'w') as f:
        json.dump(data, f)
    chrome.close()
if __name__ == "__main__":
    main()

import zipfile
​
​
def proxies(username, password, endpoint, port):
   manifest_json = """
   {
       "version": "1.0.0",
       "manifest_version": 2,
       "name": "Proxies",
       "permissions": [
           "proxy",
           "tabs",
           "unlimitedStorage",
           "storage",
           "<all_urls>",
           "webRequest",
           "webRequestBlocking"
       ],
       "background": {
           "scripts": ["background.js"]
       },
       "minimum_chrome_version":"22.0.0"
   }
   """
​
   background_js = """
   var config = {
           mode: "fixed_servers",
           rules: {
             singleProxy: {
               scheme: "http",
               host: "%s",
               port: parseInt(%s)
             },
             bypassList: ["localhost"]
           }
         };
   chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
   function callbackFn(details) {
       return {
           authCredentials: {
               username: "%s",
               password: "%s"
           }
       };
   }
   chrome.webRequest.onAuthRequired.addListener(
               callbackFn,
               {urls: ["<all_urls>"]},
               ['blocking']
   );
   """ % (endpoint, port, username, password)
​
   extension = 'proxies_extension.zip'
​
   with zipfile.ZipFile(extension, 'w') as zp:
       zp.writestr("manifest.json", manifest_json)
       zp.writestr("background.js", background_js)
​
   return extension