from urllib.request import urlopen
from urllib.request import urlretrieve
from urllib.error import HTTPError

page = urlopen(URL)

import requests

r = requests.get(URL)
r = requests.post(URL)

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
    'Accept': 'text/html, application/xhtml+xml, application/xml; q=0.9, img/webp,*/*; q=0.8',
    'Host': 'www.zhihu.com',
    'Referer': 'https://www.zhihu.com/'}
r = requests.get(URL, headers=headers)

cookies = dict(cookies_are='working')
r = requests.get(URL, cookies=cookies)
# using cookie jar
jar = requests.cookies.RequestsCookieJar()
jar.set('tasty_cookie', 'yum', domain='httpbin.org', path='/cookies')
jar.set('gross_cookie', 'blech', domain='httpbin.org', path='/elsewhere')
r = requests.get(url, cookies=jar)

r.status_code
r.headers
r.text

s = requests.session(URL)

data = "some string"
r = requests.post(URL, data=data)

import json
data = json.dumps({key1:item1, key2:item2})
r = requests.post(URL, data=data)

files = {"files":("test.pdf", open("test.pdf", "rb"),"application/pdf"), 
        "info":(None, "information")}

report = session.get(url, stream=True)
with open("report.pdf", "wb") as f:
    for chunk in report.iter_content(chunk_size=128):
        f.write(chunk)

proxies = {'http':'http://proxy:port',
    'https':'https://proxy:port'
    }
r = requests.get(URL, proxies = proxies)

from bs4 import BeautifulSoup

bsObj = BeautifulSoup(page.read(), 'html.parser')

targets = bsObj.findAll("span", {"class":"a_cover"})

from lxml import etree

page = etree.HTML(response.text)

# select all subelements beneath the current node ("//") with the given tag ("span") which the given attribute ("[@class]") has the given value ("className")
elems = page.xpath('//span[@class="className"]')
# select all elements that has a child element named tag
elems = page.xpath('[tag]')
for elem in elems:
    # get the tag, text, and attribute of an element
    print(item.tag, item.text, item.get('attrib'))
# get the text directly
elems = page.xpath('//span[@class="className"]/text()')
# position
elems = page.xpath('//span[location()=1]')

pip install selenium

# linux 需要安装chromium driver
sudo apt-get install chromium-chromedriver

from selenium import webdriver

# phantomjs 在 selenium 4.0 中被废弃，替代方法是使用chrome/firefox的headless模式
driver = webdriver.PhantomJS(executable_path=path)

# headless chrome
from selenium.webdriver.chrome.options import Options
options = Options()
options.headless = True
driver = webdriver.Chrome(options=options)

driver.get(URL)

webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.settings.userAgent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Accept-Language'] = 'en-US,en;q=0.6'

al = driver.switch_to.alert
al.accept()

from selenium.webdriver.common.by import By

driver.find_elements(by=By.CLASS_NAME, value="cell-type")
driver.find_element(By.XPATH, '//button[text()="Some text"]')

import gevent
import gevent.queue
import gevent.pool
import gevent.monkey

gevent.monkey.patch_all()

gevent_pool = gevent.pool.Pool(20)

threads = []
threads.append(gevent.spawn(function, *args))
gevent.joinall(threads)

import random
import time
time.sleep(random.random())

from urllib.parse import quote
s = quote(s)
from urllib.parse import quote