mirror of
https://github.com/go2coding/CatwalkGlimpse-AISelections.git
synced 2024-11-25 16:25:32 +08:00
81 lines
2.3 KiB
Python
81 lines
2.3 KiB
Python
# coding:utf-8
|
|
|
|
from datetime import datetime
|
|
import codecs
|
|
import requests
|
|
import os
|
|
from pyquery import PyQuery as pq
|
|
import urllib.parse
|
|
|
|
dir_path = "./data/trending/"
|
|
|
|
# 获取今天的日期
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
year = today.split("-")[0]
|
|
month = today.split("-")[1]
|
|
|
|
|
|
today_path = dir_path + year + '/' + month
|
|
|
|
|
|
def createMarkdown(date, filename):
|
|
|
|
if not os.path.isdir(today_path):
|
|
os.makedirs(today_path)
|
|
|
|
with open(today_path + '/' + filename, 'w') as f:
|
|
f.write("## " + date + "\n")
|
|
|
|
|
|
def scrape(language, filename):
|
|
HEADERS = {
|
|
'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0',
|
|
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
'Accept-Encoding' : 'gzip,deflate,sdch',
|
|
'Accept-Language' : 'zh-CN,zh;q=0.8'
|
|
}
|
|
url = 'https://github.com/trending/{language}'.format(language=urllib.parse.quote(language))
|
|
r = requests.get(url, headers=HEADERS)
|
|
if r.status_code != 200:
|
|
return
|
|
|
|
d = pq(r.content)
|
|
items = d('div.Box article.Box-row')
|
|
|
|
# codecs to solve the problem utf-8 codec like chinese
|
|
with codecs.open(today_path + '/' + filename, "a", "utf-8") as f:
|
|
f.write('\n#### {language}\n'.format(language=language))
|
|
|
|
for item in items:
|
|
i = pq(item)
|
|
title = i(".lh-condensed a").text()
|
|
owner = i(".lh-condensed span.text-normal").text()
|
|
description = i("p.col-9").text()
|
|
url = i(".lh-condensed a").attr("href")
|
|
url = "https://github.com" + url
|
|
# ownerImg = i("p.repo-list-meta a img").attr("src")
|
|
# print(ownerImg)
|
|
f.write(u"* [{title}]({url}):{description}\n".format(title=title, url=url, description=description))
|
|
|
|
|
|
|
|
def main():
|
|
strdate = datetime.now().strftime('%Y-%m-%d')
|
|
filename = '{date}.md'.format(date=strdate)
|
|
|
|
# create markdown file
|
|
createMarkdown(strdate, filename)
|
|
|
|
languages = ['c','c#','c++','java','Kotlin','Objective-C','php','rust','python','swift','javascript','go','TypeScript','Vue']
|
|
|
|
# write markdown
|
|
for language in languages:
|
|
scrape(language, filename)
|
|
|
|
# git add commit push
|
|
# git_add_commit_push(strdate, filename)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|