🚈采用fastAPI实现后端,放弃Flask
This commit is contained in:
parent
36a9140dac
commit
f4e7750bd2
@ -1,2 +1,2 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
nohup python server.py > grab.log 2>&1 &
|
nohup python3 server.py > grab.log 2>&1 &
|
||||||
|
Binary file not shown.
Binary file not shown.
@ -207,6 +207,7 @@ def fetch_and_process_data(json_url, count=5):
|
|||||||
error_friends = 0
|
error_friends = 0
|
||||||
total_articles = 0
|
total_articles = 0
|
||||||
article_data = []
|
article_data = []
|
||||||
|
error_friends_info = []
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||||
future_to_friend = {
|
future_to_friend = {
|
||||||
@ -224,9 +225,11 @@ def fetch_and_process_data(json_url, count=5):
|
|||||||
total_articles += len(result['articles'])
|
total_articles += len(result['articles'])
|
||||||
else:
|
else:
|
||||||
error_friends += 1
|
error_friends += 1
|
||||||
|
error_friends_info.append(friend)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"处理 {friend} 时发生错误: {e}")
|
print(f"处理 {friend} 时发生错误: {e}")
|
||||||
error_friends += 1
|
error_friends += 1
|
||||||
|
error_friends_info.append(friend)
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
'statistical_data': {
|
'statistical_data': {
|
||||||
@ -242,7 +245,7 @@ def fetch_and_process_data(json_url, count=5):
|
|||||||
print("数据处理完成")
|
print("数据处理完成")
|
||||||
print("总共有 %d 位朋友,其中 %d 位博客可访问,%d 位博客无法访问" % (total_friends, active_friends, error_friends))
|
print("总共有 %d 位朋友,其中 %d 位博客可访问,%d 位博客无法访问" % (total_friends, active_friends, error_friends))
|
||||||
|
|
||||||
return result
|
return result, error_friends_info
|
||||||
|
|
||||||
def sort_articles_by_time(data):
|
def sort_articles_by_time(data):
|
||||||
"""
|
"""
|
||||||
|
3
grab.log
Normal file
3
grab.log
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
2024-07-22 10:21:56,649 - INFO - 开始抓取文章...
|
||||||
|
2024-07-22 10:21:56,653 - INFO - 正在从 https://blog.qyliu.top/friend.json 中获取,每个博客获取 5 篇文章
|
||||||
|
2024-07-22 10:22:32,855 - INFO - 文章抓取成功
|
2
run.py
2
run.py
@ -15,7 +15,7 @@ if config["spider_settings"]["enable"]:
|
|||||||
json_url = config['spider_settings']['json_url']
|
json_url = config['spider_settings']['json_url']
|
||||||
article_count = config['spider_settings']['article_count']
|
article_count = config['spider_settings']['article_count']
|
||||||
print("正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章".format(json_url=json_url, article_count=article_count))
|
print("正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章".format(json_url=json_url, article_count=article_count))
|
||||||
result = fetch_and_process_data(json_url=json_url, count=article_count)
|
result, _ = fetch_and_process_data(json_url=json_url, count=article_count)
|
||||||
sorted_result = sort_articles_by_time(result)
|
sorted_result = sort_articles_by_time(result)
|
||||||
with open("all.json", "w", encoding="utf-8") as f:
|
with open("all.json", "w", encoding="utf-8") as f:
|
||||||
json.dump(sorted_result, f, ensure_ascii=False, indent=2)
|
json.dump(sorted_result, f, ensure_ascii=False, indent=2)
|
||||||
|
74
server.py
74
server.py
@ -1,21 +1,19 @@
|
|||||||
from flask import Flask, jsonify
|
from fastapi import FastAPI
|
||||||
from flask_apscheduler import APScheduler
|
from fastapi.responses import JSONResponse, HTMLResponse
|
||||||
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import json
|
||||||
|
import random
|
||||||
|
|
||||||
from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time
|
from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time
|
||||||
from friend_circle_lite.get_conf import load_config
|
from friend_circle_lite.get_conf import load_config
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = FastAPI()
|
||||||
|
|
||||||
# 配置APScheduler
|
# 配置APScheduler
|
||||||
class Config:
|
scheduler = AsyncIOScheduler()
|
||||||
SCHEDULER_API_ENABLED = True
|
|
||||||
|
|
||||||
app.config.from_object(Config())
|
|
||||||
scheduler = APScheduler()
|
|
||||||
scheduler.init_app(app)
|
|
||||||
scheduler.start()
|
scheduler.start()
|
||||||
|
|
||||||
# 配置日志记录
|
# 配置日志记录
|
||||||
@ -23,11 +21,15 @@ log_file = "grab.log"
|
|||||||
logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
logging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||||
|
|
||||||
# 全局变量
|
# 全局变量
|
||||||
articles_data = []
|
articles_data = {
|
||||||
|
"statistical_data": {},
|
||||||
|
"article_data": []
|
||||||
|
}
|
||||||
|
error_friends_info = []
|
||||||
data_lock = Lock()
|
data_lock = Lock()
|
||||||
|
|
||||||
def fetch_articles():
|
def fetch_articles():
|
||||||
global articles_data
|
global articles_data, error_friends_info
|
||||||
logging.info("开始抓取文章...")
|
logging.info("开始抓取文章...")
|
||||||
config = load_config("./conf.yaml")
|
config = load_config("./conf.yaml")
|
||||||
if config["spider_settings"]["enable"]:
|
if config["spider_settings"]["enable"]:
|
||||||
@ -35,27 +37,65 @@ def fetch_articles():
|
|||||||
article_count = config['spider_settings']['article_count']
|
article_count = config['spider_settings']['article_count']
|
||||||
logging.info(f"正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章")
|
logging.info(f"正在从 {json_url} 中获取,每个博客获取 {article_count} 篇文章")
|
||||||
try:
|
try:
|
||||||
result = fetch_and_process_data(json_url=json_url, count=article_count)
|
result, errors = fetch_and_process_data(json_url=json_url, count=article_count)
|
||||||
sorted_result = sort_articles_by_time(result)
|
sorted_result = sort_articles_by_time(result)
|
||||||
with data_lock:
|
with data_lock:
|
||||||
articles_data = sorted_result
|
articles_data = sorted_result
|
||||||
|
error_friends_info = errors
|
||||||
logging.info("文章抓取成功")
|
logging.info("文章抓取成功")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"抓取文章时出错: {e}")
|
logging.error(f"抓取文章时出错: {e}")
|
||||||
|
|
||||||
# 每四个小时抓取一次文章
|
# 每四个小时抓取一次文章
|
||||||
scheduler.add_job(id='Fetch_Articles_Job', func=fetch_articles, trigger='interval', hours=4)
|
scheduler.add_job(fetch_articles, 'interval', hours=4)
|
||||||
|
|
||||||
@app.route('/all', methods=['GET'])
|
@app.get("/", response_class=HTMLResponse)
|
||||||
def get_all_articles():
|
async def root():
|
||||||
|
html_content = """
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Friend Circle Lite</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>欢迎使用 Friend Circle Lite</h1>
|
||||||
|
<p>这是一个轻量版友链朋友圈,有两种部署方式,其中自部署使用 fastAPI,还有 github action 部署方式,可以很方便的从友链中获取文章并展示到前端。</p>
|
||||||
|
<ul>
|
||||||
|
<li><a href="/all">查看所有文章,按照时间进行排序</a></li>
|
||||||
|
<li><a href="/errors">查看出错误数据,包含所有的错误友链信息,可自行发挥</a></li>
|
||||||
|
<li><a href="/random">随机文章</a></li>
|
||||||
|
</ul>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
return HTMLResponse(content=html_content)
|
||||||
|
|
||||||
|
@app.get('/all')
|
||||||
|
async def get_all_articles():
|
||||||
with data_lock:
|
with data_lock:
|
||||||
return jsonify(articles_data)
|
return JSONResponse(content=articles_data)
|
||||||
|
|
||||||
|
@app.get('/errors')
|
||||||
|
async def get_error_friends():
|
||||||
|
with data_lock:
|
||||||
|
return JSONResponse(content=error_friends_info)
|
||||||
|
|
||||||
|
@app.get('/random')
|
||||||
|
async def get_random_article():
|
||||||
|
with data_lock:
|
||||||
|
if articles_data["article_data"]:
|
||||||
|
random_article = random.choice(articles_data["article_data"])
|
||||||
|
return JSONResponse(content=random_article)
|
||||||
|
else:
|
||||||
|
return JSONResponse(content={"error": "No articles available"}, status_code=404)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
# 清空日志文件
|
# 清空日志文件
|
||||||
if os.path.exists(log_file):
|
if os.path.exists(log_file):
|
||||||
with open(log_file, 'w'):
|
with open(log_file, 'w'):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
fetch_articles() # 启动时立即抓取一次
|
fetch_articles() # 启动时立即抓取一次
|
||||||
app.run(port=1223)
|
uvicorn.run(app, host='0.0.0.0', port=1223)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user