Selenium User Manual

I. Installation and environment configuration

To use Selenium, you need to download Chrome driver for the corresponding operating system and browser version. PIP Install Selenium

2.1 Startup and Configuration

# Normal boot

from selenium import webdriver

# 1. Instantiate a browser
browser = webdriver.Chrome("./chromedriver")  The parameter is the driver path

browser.get("http://www.baidu.com/")  # Drive browser to visit Baidu

browser.page_source  Get the HTML of the current browser

browser.quit()  Exit the instantiated browser
Copy the code

Start with configuration information

from selenium import webdriver

# configuration
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # Added no interface option
options.add_argument('--disable-gpu')  If you do not add this option, sometimes positioning problems will occur
options.add_argument('--user-data-dir=/Users/mac/Library/Application Support/Google/Chrome/Default')  Set it to the user's own data directory

Instantiate a browser object and load the configuration
browser = webdriver.Chrome(executable_path="./chromedriver", options=options)

browser.get("http://www.baidu.com/")  # Drive browser to visit Baidu

browser.page_source  Get the HTML of the current browser

browser.quit()  Exit the instantiated browser
Copy the code

# locate

browser.find_element_by_id("id")  Location by id
browser.find_element_by_name("name")  # locate by name
browser.find_element_by_class_name("class_name")  # locate by classname
browser.find_element_by_tag_name("input")  # Locate by using the tag
browser.find_element_by_xpath("//input[@id='kw']")  Location via xpath
browser.find_element_by_css_selector("#kw")  # Using THE CSS
Copy the code

# wait

# 1. Force wait x seconds
time.sleep(10)  Force the program to sleep for 10 seconds

# 2. Wait until the browser is fully loaded each time
browser.implicitly_wait(30)  # Implicit wait, parameter is maximum wait time (s). Go to the next step only after the page is loaded.
Browser.implicitly_wait (30) sets the maximum wait time for the whole world to 30s.

# 3. Wait until the elements you need appear or the ones you don't need disappear
from selenium.webdriver.support.wait import WebDriverWait

WebDriverWait(browser, 30).until(EC.title_is("Just Google it and you'll know."))  # wait until the title is baidu, or timeout
WebDriverWait(browser, 30).until(EC.title_contains("Baidu"))  Wait until the title contains Baidu, or time out
WebDriverWait(browser, 30).until(EC.presence_of_element_located((By.ID, "id")))  Wait until the ID exists in the Dom
WebDriverWait(browser, 30).until(EC.visibility_of_element_located((By.ID, "id")))  Wait until the ID exists in the Dom and is visible
WebDriverWait(browser, 30).until(EC.element_to_be_clickable((By.XPATH,"//*[@id='u1']/a[8]")))  Wait until the element is clickable
WebDriverWait(browser, 30).until(EC.text_to_be_present_in_element_value((By.CSS_SELECTOR,'.main'),'enable'))  # check whether.main contains the enable string, returning a Boolean value
WebDriverWait(browser, 30).until(EC.staleness_of(driver.find_element(By.ID,'id')))  Wait until the ID is removed from the DOM
Copy the code

# behavior

browser.maximize_window()  # browser maximization
browser.minimize_window()  # Browser minimization
browser.set_window_size(480.800)  Customize the browser window size
browser.forword()  # browser Forward
browser.back()  # Browser Back
browser.close()  Close the current browser window
browser.quit()  Exit the browser driver and close all Windows
browser.refresh()  Refresh the current page
browser.page_source  Get the current page HTML
browser.title  Get the current page title
browser.url  Get the current page URL

browser.find_element_by_name("name").click()  # Click on object
browser.find_element_by_name("name").send_keys("keyword")  # Simulate keyboard input
browser.find_element_by_name("name").clear()  Clear the contents of the object
browser.find_element_by_name("name").submit()  Submit the contents of the object
browser.find_element_by_name("name").text()  Get the text information for the element
browser.find_element_by_name("name").context_click()  # right-click
browser.find_element_by_name("name").double_click()  # double-click
browser.find_element_by_name("name").is_displayed()  # Visible to the user


# Simulate keyboard events
from selenium.webdriver.common.keys import Keys
browser.find_element_by_id("user_pwd").send_keys(Keys.ENTER)  # Replace the login button by locating the password box and entering
browser.find_element_by_id("login").send_keys(Keys.ENTER)  # Click enter to log in, same as above
browser.find_element_by_name("name").send_keys(Keys.CONTROL, "a")  CTRL + a # simulation
Copy the code

# iframe

browser.switch_to_frame("f1")  # find f1 iframe
browser.switch_to_window("f1")  # find the embedded window f1
driver.switch_to.frame(driver.find_elements_by_tag_name("iframe") [0])
Copy the code

Other uses

# operating cookie

browser.get_cookie("name")  Get a single cookie based on name
browser.get_cookies()  Get all cookies
browser.delete_all_cookies()  # Delete all cookies
browser.delete_cookie("name")  Delete cookie based on name
browser.add_cookie({"k1": "v1"."k2": "v2"})  # set cookies
Copy the code

Get a screenshot

browser.get_screenshot_as_file("/usr/download/down_image.png")  Save the screenshot of the current window
browser.get_screenshot_as_png()  Get the binary string of the current PNG screenshot
Copy the code

Get window information

browser.get_window_rect()  Get the height and width of the current window
browser.get_window_position(windowHandle="current")  Get the x,y coordinates of the current window
browser.get_window_size(windowHandle="current")  Get the height and width of the current window
Copy the code

# Multi-window switch

driver.get("http://example.com")  Open a window
now_handle = drvier.current_window_handle  Get the handle to the current window

driver.find_element_by_name('example').click()  # Click an element to open a new window (target="_black" element)
all_handle = drvier.window_handles  Get all window handles
drvier.switch_to.window(now_handle)  Switch to the first window
driver.close()  Close the current window
Copy the code

Upload attachments via input
self.browser.find_element_by_xpath('//input[@class="file-selector-file"]').send_keys("/Users/mac/Documents/myspider/selenium_spider/test.mp4")  Absolute path must be used
Copy the code

# 1. Get a list of handles to all current tabs
current_windows = driver.window_handles
 
# 2. Switch according to the index subscript of the TAB handle list
driver.switch_to.window(current_windows[0])
Copy the code

3.2 switch iframe

The selenium operation iframe content needs to be switch_to iframe first, and switch back after the operation is complete

login_frame = driver.find_element_by_id('login_frame') Position the frame element by ID
driver.switch_to.frame(login_frame) Go to the frame

Cut out the frame tag by switching tabs
windows = driver.window_handles
driver.switch_to.window(windows[0])
Copy the code

# get cookie
cookies_dict = {cookie["name"]: cookie["value"] for cookie in driver.get_cookies()}

Delete a cookie
driver.delete_cookie("CookieName")

Delete all cookies
driver.delete_all_cookies()

Add a cookie
add_cookie(cookie_dict)
Copy the code

3.4 Executing JS Code

script = 'window.scrollTo(0,document.body.scrollHeight)'

browser.execute_async_script(script, *args)  Execute JS code asynchronously in the current window/frame
browser.execute_script(script, *args)  # Execute JS code synchronously in the current window/frameScript: JS code (strArgs: Arguments to be passed to JS (iterable)Copy the code

self.browser.find_element_by_xpath('//input[@class="file-selector-file"]').send_keys("/Users/mac/Documents/myspider/selenium_spider/test.mp4")  Absolute path must be used
Copy the code

3.6 screenshot

browser.get_screenshot_as_file("/usr/download/down_image.png")  Save the screenshot of the current window
browser.get_screenshot_as_png()  Get the binary string of the current PNG screenshot
Copy the code

from selenium.webdriver.common.action_chains import ActionChains

 el = driver.find_element_by_name('tj_trnews')  # Target element

ActionChains(driver).context_click(el).perform()  # right-click the target element
ActionChains(driver).double_click(el).perform()  Double-click the target element

source = driver.find_element_by_id('lg')   The target element's original location
target = driver.find_element_by_id('kw')  Drag the target position
ActionChains(driver).drag_and_drop(source, target).perform()  # Drag element

ActionChains(driver).move_to_element(el).perform()  # mouse over target element
ActionChains(driver).click_and_hold(el).perform()  # Move to the target element and press the left mouse button
Copy the code

3.8 Controlling an Open Browser

.\chrome.exe --remote-debugging-port=6001 --user-data-dir="C:\ProgramFiles\Chrome"

def init_driver() :
    options = webdriver.ChromeOptions()
    options.add_experimental_option("debuggerAddress"."127.0.0.1:6001")
    driver = webdriver.Chrome(options=options)
    return driver
Copy the code

4.1 Initializing the Driver

#! /usr/bin/env python
# coding:utf-8
import time

from selenium import webdriver
from selenium.webdriver.support.select import Select


def chromedriver_demo() :
    driver = webdriver.Chrome()

    driver.delete_all_cookies()

    driver.add_cookie({'name': 'ABC'.'value': 'DEF'})

    driver.get_cookies()

    driver.execute_script('window.open("https://www.baidu.com"); ')

    """ // Chrome address bar command about:version - displays current version about:memory - displays native browser memory usage about:plugins - displays installed plug-ins about:histograms - Display history about: DNS - Display DNS status about:cache - Display cache page About: GPU - Whether hardware acceleration is enabled about: Flags - Enable some plug-ins Chrome ://extensions/ - View installed extensions/ / Chrome parameter -- user-data-dir= "[PATH]" specifies the user data PATH to the user folder. You can save user data such as bookmarks in a partition other than the system partition. - disk-cache-dir= "[PATH]" Specify cache cache PATH - disk-cache-size= Specify cache size, in Byte - first run to reset to the initial state. Run for the first time - Incognito incognito mode enable - disable-javascript Disable javascript - Omnibox-popup -count= "num" Change the number of prompt menus displayed in the address bar to num. I made it 15. - user-agent= "XXXXXXXX" Modify the agent string in the header of HTTP requests. You can view the modification effect on the About :version page. - Disable-plugins Disable the loading of all plug-ins to increase the speed. You can see the effect from the About :plugins page -- disable-javascript Disables javascript, Add this -- disable-java to disable Java -- start-maximized startup to maximize -- no-sandbox to disable sandbox mode -- single-process to run a single process - process-per-tab Uses a separate process for each label - process-per-site Uses a separate process for each site - In-process-plugins does not enable a separate process- disable-popup-blocking Disable pop-up blocking - disable-plugins disable plug-ins - disable-images Disable images - incognito enables incognito mode - enable-udd-profiles Enables the account switching menu - proxy-pac-url Uses pac proxy [via 1/2] - lang= zh-cn Set the language to simplified Chinese - disk-cache-dir User-defined cache directory - disk-cache-size - Media-cache-size Specifies the maximum value of the user-defined multimedia cache (byte). - bookmark-menu Adds a bookmark button on the toolbar. - enable-sync Enables bookmark synchronization - Single-process When a single process runs Google Chrome - start-maximized When Google Chrome is started - disable-java Disables Java - no-sandbox Run in non-sandbox mode.


def init_driver(driver_path: str, timeout=20, user_agent: str = None, headless=False,
                proxy: str = None, binary_location: str = None, user_data_dir: str = None,
                crx_dir=None) :
    chrome_options = webdriver.ChromeOptions()

    chrome_options.add_argument("blink-settings=imagesEnabled=false")  # set image not to load
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("disable-infobars")  # Hide "Chrome is being controlled by automated software"
    chrome_options.add_argument("lang=zh_CN.UTF-8")  # Set Chinese
    chrome_options.add_argument("window-size=1920x3000")  # specify browser resolution
    chrome_options.add_argument("--hide-scrollbars")  # Hide the scroll bar for some special pages
    chrome_options.add_argument("--remote-debugging-port=9222")
    # chrome_options.binary_location = r"/Applications/Chrome

    if headless:
        chrome_options.add_argument("--headless")

    if not user_agent:
        user_agent = "Mozilla / 5.0 (Windows NT 10.0; Win64; X64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/ 537.36edg /86.0.622.58"
        chrome_options.add_argument("user-agent=%s" % user_agent)

    if proxy:
        chrome_options.add_argument("proxy-server=%s" % proxy)

    if user_data_dir:
        chrome_options.add_argument("--user-data-dir=%s" % user_data_dir)

    if binary_location:
        chrome_options.binary_location = binary_location  # Manually specify the browser location to use

    if crx_dir:
        chrome_options.add_extension(crx_dir)  # Custom load extension CRX

    driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_path)
    driver.set_page_load_timeout(timeout)
    driver.set_script_timeout(timeout)
    return driver


def init_driver_with_exists_chrome() :
    """ Start Chrome: C:\Users\user\AppData\Local\Google\Chrome\Application\chrome.exe --remote-debugging-port=6001 --user-data-dir="C:\ProgramFiles\Chrome2" """
    options = webdriver.ChromeOptions()
    options.add_experimental_option("debuggerAddress"."127.0.0.1:6001")
    driver = webdriver.Chrome(options=options)
    return driver


def wait_html(driver) :
    while 1:
        time.sleep(0.5)
        if "Certification" in driver.page_source:
            print("Success!!! Certification")
            break
        print("Certification")

    # pass. to do next


def auto_input(driver) :
    # input keys
    driver.find_element_by_xpath('//input[@name="CMT_SSN_1"]').send_keys("")

    # click button 1
    driver.find_element_by_xpath('//input[@name="CCA_QUIT_IND"][@value="N"]').click()
    # click button 2 (with js)
    driver.execute_script(""" var e = document.createEvent("MouseEvents"); e.initEvent("click", true, true); document.getElementById("nextBtn").dispatchEvent(e); "" ")
    # click button 3
    element = driver.find_element_by_id('login')
    webdriver.ActionChains(driver).move_to_element(element).click(element).perform()

    # selection
    Select(driver.find_element_by_xpath('//select[@name="CMT_ID_TYPE_CD"]')).select_by_index(2)
    Select(driver.find_element_by_xpath('//select[@name="CWE_CONTACT_OUTCOME_CD_02"]')).select_by_value("3")


if __name__ == "__main__":
    driver_path = "spider_data/chromedriver.exe"
    init_driver(driver_path)

Copy the code