🚈采用fastAPI实现后端,放弃Flask

This commit is contained in:
柳神 2024-07-22 10:55:44 +08:00
parent 36a9140dac
commit f4e7750bd2
7 changed files with 66 additions and 20 deletions

View File

@ -1,2 +1,2 @@
#!/bin/bash #!/bin/bash
nohup python server.py > grab.log 2>&1 & nohup python3 server.py > grab.log 2>&1 &

View File

@ -207,6 +207,7 @@ def fetch_and_process_data(json_url, count=5):
error_friends = 0 error_friends = 0
total_articles = 0 total_articles = 0
article_data = [] article_data = []
error_friends_info = []
with ThreadPoolExecutor(max_workers=10) as executor: with ThreadPoolExecutor(max_workers=10) as executor:
future_to_friend = { future_to_friend = {
@ -224,9 +225,11 @@ def fetch_and_process_data(json_url, count=5):
total_articles += len(result['articles']) total_articles += len(result['articles'])
else: else:
error_friends += 1 error_friends += 1
error_friends_info.append(friend)
except Exception as e: except Exception as e:
print(f"处理 {friend} 时发生错误: {e}") print(f"处理 {friend} 时发生错误: {e}")
error_friends += 1 error_friends += 1
error_friends_info.append(friend)
result = { result = {
'statistical_data': { 'statistical_data': {
@ -242,7 +245,7 @@ def fetch_and_process_data(json_url, count=5):
print("数据处理完成") print("数据处理完成")
print("总共有 %d 位朋友,其中 %d 位博客可访问,%d 位博客无法访问" % (total_friends, active_friends, error_friends)) print("总共有 %d 位朋友,其中 %d 位博客可访问,%d 位博客无法访问" % (total_friends, active_friends, error_friends))
return result return result, error_friends_info
def sort_articles_by_time(data): def sort_articles_by_time(data):
""" """

3
grab.log Normal file
View File

@ -0,0 +1,3 @@
2024-07-22 10:21:56,649 - INFO - 开始抓取文章...
2024-07-22 10:21:56,653 - INFO - 正在从 https://blog.qyliu.top/friend.json 中获取,每个博客获取 5 篇文章
2024-07-22 10:22:32,855 - INFO - 文章抓取成功

2
run.py
View File

@ -15,7 +15,7 @@ if config["spider_settings"]["enable"]:
json_url = config['spider_settings']['json_url'] json_url = config['spider_settings']['json_url']
article_count = config['spider_settings']['article_count'] article_count = config['spider_settings']['article_count']
print("正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章".format(json_url=json_url, article_count=article_count)) print("正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章".format(json_url=json_url, article_count=article_count))
result = fetch_and_process_data(json_url=json_url, count=article_count) result, _ = fetch_and_process_data(json_url=json_url, count=article_count)
sorted_result = sort_articles_by_time(result) sorted_result = sort_articles_by_time(result)
with open("all.json", "w", encoding="utf-8") as f: with open("all.json", "w", encoding="utf-8") as f:
json.dump(sorted_result, f, ensure_ascii=False, indent=2) json.dump(sorted_result, f, ensure_ascii=False, indent=2)

View File

@ -1,21 +1,19 @@
from flask import Flask, jsonify from fastapi import FastAPI
from flask_apscheduler import APScheduler from fastapi.responses import JSONResponse, HTMLResponse
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from threading import Lock from threading import Lock
import logging import logging
import os import os
import json
import random
from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time
from friend_circle_lite.get_conf import load_config from friend_circle_lite.get_conf import load_config
app = Flask(__name__) app = FastAPI()
# 配置APScheduler # 配置APScheduler
class Config: scheduler = AsyncIOScheduler()
SCHEDULER_API_ENABLED = True
app.config.from_object(Config())
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start() scheduler.start()
# 配置日志记录 # 配置日志记录
@ -23,11 +21,15 @@ log_file = "grab.log"
logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
# 全局变量 # 全局变量
articles_data = [] articles_data = {
"statistical_data": {},
"article_data": []
}
error_friends_info = []
data_lock = Lock() data_lock = Lock()
def fetch_articles(): def fetch_articles():
global articles_data global articles_data, error_friends_info
logging.info("开始抓取文章...") logging.info("开始抓取文章...")
config = load_config("./conf.yaml") config = load_config("./conf.yaml")
if config["spider_settings"]["enable"]: if config["spider_settings"]["enable"]:
@ -35,27 +37,65 @@ def fetch_articles():
article_count = config['spider_settings']['article_count'] article_count = config['spider_settings']['article_count']
logging.info(f"正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章") logging.info(f"正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章")
try: try:
result = fetch_and_process_data(json_url=json_url, count=article_count) result, errors = fetch_and_process_data(json_url=json_url, count=article_count)
sorted_result = sort_articles_by_time(result) sorted_result = sort_articles_by_time(result)
with data_lock: with data_lock:
articles_data = sorted_result articles_data = sorted_result
error_friends_info = errors
logging.info("文章抓取成功") logging.info("文章抓取成功")
except Exception as e: except Exception as e:
logging.error(f"抓取文章时出错: {e}") logging.error(f"抓取文章时出错: {e}")
# 每四个小时抓取一次文章 # 每四个小时抓取一次文章
scheduler.add_job(id='Fetch_Articles_Job', func=fetch_articles, trigger='interval', hours=4) scheduler.add_job(fetch_articles, 'interval', hours=4)
@app.route('/all', methods=['GET']) @app.get("/", response_class=HTMLResponse)
def get_all_articles(): async def root():
html_content = """
<!DOCTYPE html>
<html>
<head>
<title>Friend Circle Lite</title>
</head>
<body>
<h1>欢迎使用 Friend Circle Lite</h1>
<p>这是一个轻量版友链朋友圈有两种部署方式其中自部署使用 fastAPI还有 github action 部署方式可以很方便的从友链中获取文章并展示到前端</p>
<ul>
<li><a href="/all">查看所有文章按照时间进行排序</a></li>
<li><a href="/errors">查看出错误数据包含所有的错误友链信息可自行发挥</a></li>
<li><a href="/random">随机文章</a></li>
</ul>
</body>
</html>
"""
return HTMLResponse(content=html_content)
@app.get('/all')
async def get_all_articles():
with data_lock: with data_lock:
return jsonify(articles_data) return JSONResponse(content=articles_data)
@app.get('/errors')
async def get_error_friends():
with data_lock:
return JSONResponse(content=error_friends_info)
@app.get('/random')
async def get_random_article():
with data_lock:
if articles_data["article_data"]:
random_article = random.choice(articles_data["article_data"])
return JSONResponse(content=random_article)
else:
return JSONResponse(content={"error": "No articles available"}, status_code=404)
if __name__ == '__main__': if __name__ == '__main__':
import uvicorn
# 清空日志文件 # 清空日志文件
if os.path.exists(log_file): if os.path.exists(log_file):
with open(log_file, 'w'): with open(log_file, 'w'):
pass pass
fetch_articles() # 启动时立即抓取一次 fetch_articles() # 启动时立即抓取一次
app.run(port=1223) uvicorn.run(app, host='0.0.0.0', port=1223)