A simple idea occurred to me: since you can’t download pictures from the official website, I’ll help you collect them and send them to you. (Collecting data, just write a crawler?)

So I went straight to the NASA website to analyze a wave of requests. The results… Well, I’m one of them, and I can’t load an image.

How could I shrink back from this difficulty? Then, I went to the comments section of Weibo to look for it. Sure enough, I found a big guy on Douban who had already collected all the pictures:

In the spirit of “fetch”, I decided that this would be my data source (Douban photo album)

Simple analysis, found that can flip through a m_start parameters, and each page 20 images (such as m_start = 0 for the first page, m_start = 20 for the second page), then write a loop can be:

import re
import queue
import requests
import threading
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
 
headers = {
'Host': 'www.douban.com'.'Connection': 'keep-alive'.'Cache-Control': 'max-age=0'.'Upgrade-Insecure-Requests': '1'.'User-Agent': 'the Mozilla / 5.0 (Windows NT 6.3; Win64; X64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'.'Sec-Fetch-Dest': 'document'.'Accept': 'text/html,application/xhtml+xml,application/xml; Q = 0.9, image/webp image/apng, * / *; Q = 0.8, application/signed - exchange; v=b3; Q = 0.9 '.'Sec-Fetch-Site': 'none'.'Sec-Fetch-Mode': 'navigate'.'Sec-Fetch-User': '? 1 '.'Accept-Encoding': 'gzip, deflate, br'.'Accept-Language': 'zh-CN,zh; Q = 0.9 '.'Cookie': 'bid=rb_kUqiDS6k; douban-fav-remind=1; _pk_ses. 100001.8 cb4 = *; Ap_v = 0,6.0; __utma = 30149280.1787149566.1585488263.1585488263.1585488263.1; __utmc=30149280; __utmz = 30149280.1585488263.1.1. Utmcsr = (direct) | utmccn = (direct) | utmcmd = (none); __yadk_uid=HNoH1YVIvD2c8HrQDWHRzyLciFJl1AVD; __gads=ID=a1f73d5d4aa31261:T=1585488663:S=ALNI_MafqKPZWHx0TGWTpKEm8TTvdC-eyQ; ct=y; _pk_id. 100001.8 cb4 = 722 e0554d0127ce7. 1585488261.1.1585488766.1585488261.; __utmb = 30149280.10.6.1585488263 '
}
 
# driver initialization
chrome_options = Options()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome(options=chrome_options)
 
# Download image
def downimg() :
    while not img_queue.empty():
        img = img_queue.get()
        img_name = img[0]
        url = img[1]
        res = requests.get(url)
        data =res.content
        with open('./img/%s.webp'%img_name,'wb') as f:
            f.write(data)
        print(img_name)
         
# site parameters
url_o = 'https://www.douban.com/photos/album/1872547715/?m_start=%d'
 
# crawl the connection
img_queue = queue.Queue()
for i in range(0.21):
    url = url_o%(18*i)
    driver.get(url)
    es = driver.find_elements_by_class_name('photo_wrap')
    for e in es:
        img_e = e.find_element_by_tag_name('img')
        img_url = img_e.get_attribute('src')
        img_url = img_url.replace('photo/m/public'.'photo/l/public') # replace with big picture
        text_e = e.find_element_by_class_name('pl')
        img_date = text_e.text
        img_queue.put((img_date,img_url))
    print('%d page climb complete '%(i+1))
driver.close()
 
# Download image
thread_list = []
N_thread = 5
for i in range(N_thread):
    thread_list.append(threading.Thread(target=downimg))
for t in thread_list:
    t.start()
for t in thread_list:
    t.join()
Copy the code

The code is simple: WebDriver accesses the page, fetches the image address, and then uses Requests to download and save the image through multiple threads. At this point, the picture collection work is basically completed!

Although this traffic is far from what I expected, after all, this topic is also read by hundreds of millions of people, but I decided to do a simple statistics of yesterday’s visit:

Since the table contains not only data from NASA pages, but also data from other pages, it is necessary to filter the data:

# fetch data
data = pd.read_csv('./analyze/20200330-20200330.csv',encoding='utf-8')
 
# Filter data (NASA-related data with expiration date)
data_NASA = []
for i in range(len(data)):
    url = urllib.parse.unquote(data['URL'][i])
    pv = data['PV'][i] # views
    uv = data['UV'][i] # visitors
    #if url[-1] == 'day' and 'NaN' not in url: # for NASA access page
    if 'date=' in url and 'NaN' not in url:
        try:
            data_NASA.append((re.findall('date=(\d*? Month \ d *? ) ',url)[0],pv,uv))
        except:
            pass
Copy the code

# Judge your horoscope
def get_xingzuo(month, date) :
    dates = (21.20.21.21.22.22.23.24.24.24.23.22)
    constellations = ("Capricorn"."Aquarius"."Pisces"."Aries"."Taurus"."Gemini"."Cancer"."狮子座"."Virgo"."Libra"."Scorpio"."Sagittarius"."Capricorn")
    if date < dates[month-1] :return constellations[month-1]
    else:
        return constellations[month]
 
# Count the frequency of each constellation
xingzuo = ("Capricorn"."Aquarius"."Pisces"."Aries"."Taurus"."Gemini"."Cancer"."狮子座"."Virgo"."Libra"."Scorpio"."Sagittarius"."Capricorn")
xingzuo_map = {}
for x in xingzuo:
    xingzuo_map[x] = 0
xingzuo_total = 0
for d in data_NASA:
    month = int(re.findall('(\d*?) Month (\ d *? Days',d[0[])0] [0])
    day = int(re.findall('(\d*?) Month (\ d *? Days',d[0[])0] [1])
    x = get_xingzuo(month,day)
    #xingzuo_map[x] += d[1] # PV
    xingzuo_map[x] += d[2] # UV
    xingzuo_total += d[2]
for k in xingzuo_map.keys():
    xingzuo_map[k] = xingzuo_map[k]/xingzuo_total*100
xingzuos= sorted(xingzuo_map.items(),key=lambda x:x[1],reverse=True) # sort
Copy the code

## Birthday query TOP10- by visitor volume UV
date = []
uv = []
for i in UVs:
    date.append(i[0])
    uv.append(i[1])
top10_date = date[:10]
top10_date.reverse()
top10_uv = uv[:10]
top10_uv.reverse()
fig, ax = plt.subplots() # drawing
b = plt.barh(top10_date,top10_uv,color='#6699CC') # gold #FFFACD Silver #C0C0C0 Orange #FFA500 Blue #6699CC
i = len(b)
for rect in b: # draw numerical
    if i==3: # 3
        rect.set_facecolor('#FFA500') # orange
    if i==2: # 2
        rect.set_facecolor('#C0C0C0') # silver
    if i==1: # 1
        rect.set_facecolor('#FFFACD') # gold
    w = rect.get_width()
    ax.text(w, rect.get_y()+rect.get_height()/2.' %.2f%%'%w,ha='left', va='center')
    i -= 1
plt.xticks([]) # turn off the x-coordinate
 
 
 
## Constellation query ranking
name = []
v = []
for i in xingzuos:
    name.append(i[0])
    v.append(i[1])
name.reverse()
v.reverse()
fig, ax = plt.subplots() # drawing
b = plt.barh(name,v,color='#6699CC') # gold #FFFACD Silver #C0C0C0 Orange #FFA500 Blue #6699CC
i = len(b)
for rect in b: # draw numerical
    if i==3: # 3
        rect.set_facecolor('#FFA500') # orange
    if i==2: # 2
        rect.set_facecolor('#C0C0C0') # silver
    if i==1: # 1
        rect.set_facecolor('#FFFACD') # gold
    w = rect.get_width()
    ax.text(w, rect.get_y()+rect.get_height()/2.' %.2f%%'%w,ha='left', va='center')
    i -= 1
plt.xticks([]) # turn off the x-coordinate
 
## Month query ranking
name = []
v = []
for i in months:
    name.append(i[0])
    v.append(i[1])
name.reverse()
v.reverse()
fig, ax = plt.subplots() # drawing
b = plt.barh(name,v,color='#6699CC') # gold #FFFACD Silver #C0C0C0 Orange #FFA500 Blue #6699CC
i = len(b)
for rect in b: # draw numerical
    if i==3: # 3
        rect.set_facecolor('#FFA500') # orange
    if i==2: # 2
        rect.set_facecolor('#C0C0C0') # silver
    if i==1: # 1
        rect.set_facecolor('#FFFACD') # gold
    w = rect.get_width()
    ax.text(w, rect.get_y()+rect.get_height()/2.' %.2f%%'%w,ha='left', va='center')
    i -= 1
plt.xticks([]) # turn off the x-coordinate
Copy the code

The final result looks something like this: