【代码】发送请求响应页面

发表于 更新于 阅读次数：

前言

爬虫发送请求响应页面

无参请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
import urllib.request

def load_data():
    url = "http://www.baidu.com/"
    # response：http响应的对象
    response = urllib.request.urlopen(url)
    # print(response)
    # 读取内容 bytes类型
    data = response.read()
    # print(data)
    """
    python爬取的类型：str bytes
    如果爬取回来的是bytes类型，但是你写入的时候需要str类型：decode("utf-8")
    如果爬取回来的是str类型，但是你写入的时候需要bytes类型：encode("utf-8")
    """
    # 转换成字符串
    str_data = data.decode("utf-8")
    print(str_data)
    # 将数据写入文件
    with open("baidu.html", "w", encoding="utf-8") as f:
        f.write(str_data)

load_data()

有参请求

GET请求

字符串拼接传参

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
import urllib.request
import urllib.parse
import string

def get_method_params():
    url = "http://www.baidu.com/s?wd="
    # name = "test"
    name = "测试"
    final_url = url + name
    # python是解释性语言，解析器只支持ascii码0-127之间，不支持中文
    # 网址中包含了汉字，需要转义，转义后：http://www.baidu.com/s?wd=%E6%B5%8B%E8%AF%95
    encode_url_new = urllib.parse.quote(final_url, safe=string.printable)
    # print(final_url)
    print(encode_url_new)
    response = urllib.request.urlopen(encode_url_new)
    # print(response)
    data = response.read().decode()
    print(data)
    with open("baidu.html", "w", encoding="utf-8") as f:
        f.write(data)

get_method_params()

字典传参

  • 可以传递多个参数
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
import urllib.request
import urllib.parse
import string

def get_method_params():
    url = "http://www.baidu.com/s?"
    params = {
        "wd": "中文",
        "key": "张",
        "value": "三"
    }
    str_params = urllib.parse.urlencode(params)
    print(str_params)
    final_url = url + str_params
    print(final_url)
    # 将带有中文的url转义成可以识别的url
    end_url = urllib.parse.quote(final_url, safe=string.printable)
    response = urllib.request.urlopen(end_url)
    data = response.read().decode()
    print(data)

get_method_params()

POST请求

字符串传参

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
import urllib.request
import urllib.parse
import string

def post_method_params():
    url = "http://www.baidu.com/s"
    params = "wd=测试"
    end_url = url + params;
    # 字符串中文转换
    end_str_params = urllib.parse.quote(end_url, safe=string.printable)
    # 转bytes数组
    end_params = end_str_params.encode("utf-8")
    response = urllib.request.urlopen(url, end_params)
    data = response.read().decode()
    print(data)

post_method_params()

字典传参

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import urllib.request
import urllib.parse
import string

def post_method_params():
    url = "http://www.baidu.com/s"
    params = {
        "wd": "test"
    }
    # 字典转字符串
    str_params = urllib.parse.urlencode(params)
    # 字符串中文转换
    end_str_params = urllib.parse.quote(str_params, safe=string.printable)
    # 转bytes数组
    end_params = end_str_params.encode("utf-8")
    response = urllib.request.urlopen(url, end_params)
    data = response.read().decode()
    print(data)

post_method_params()

完成

参考文献

哔哩哔哩——随风kali